arches_extensions.management.commands.etl 

 1import json
 2
 3from django.core.management.base import BaseCommand, CommandError
 4
 5from arches_extensions.managers import ExtensionManager
 6from arches_extensions.utils import ArchesHelpTextFormatter, ArchesCLIStyles
 7
 8s = ArchesCLIStyles()
 9
10class Command(BaseCommand):
11    """
12    Run ETL modules from the command line instead of through a browser.
13
14    Usage:
15
16        python manage.py etl [module_name] [method_name] [--opts arg1=val1 arg2=val2]
17
18    A rudimentary and generic implementation so far, allows you to run a specific method on a ETL module, passing in keyword arguments as needed. This can be used during the development of ETL modules, or later to run server-side ETL operations that mimic the functionality available through the browser.
19    """
20
21    def __init__(self, *args, **kwargs):
22        self.help = self.__doc__
23
24    def add_arguments(self, parser):
25        parser.formatter_class = ArchesHelpTextFormatter
26        parser.add_argument("module_name",
27            help=f"Name of ETL module to use. Use {s.fg.pink}python manage.py extension list etl-module{s.reset} to get a list of valid ETL module names. Remember to use quotes around names that contain spaces."
28        )
29        parser.add_argument("method_name",
30            help='Name of method on ETL module class to run.'
31        )
32        parser.add_argument("--opts",
33            nargs='*',
34            help=f"Pass arbitrary keyword arguments to the method. Use format {s.opt('--opts arg1=val1 arg2=val2')}"
35        )
36
37    def handle(self, *args, **options):
38
39        try:
40            opts = {}
41            for i in options['opts']:
42                k, v = i.split("=")
43                if v == "True":
44                    v = True
45                elif v == "False":
46                    v = False
47                opts[k] = v
48        except Exception as e:
49            print(s.error(options['opts']))
50            print(s.warn("Invalid opts list. Format must be --opts arg1=val1 arg2=val2"))
51            exit()
52
53        self.run_method(options['module_name'], options['method_name'], **opts)
54
55    def print_module_list(self):
56        ExtensionManager("etl-module").print_list()
57
58    def run_method(self, module_name, method_name, **kwargs):
59
60        try:
61            instance = ExtensionManager("etl-module")._get_instance(module_name)
62        except Exception as e:
63            print(s.error(e))
64            print(s.warn("Valid module names:"))
65            self.print_module_list()
66            exit()
67
68        module_class = instance.get_class_module()
69        module = module_class()
70        try:
71            method = getattr(module, method_name)
72        except AttributeError as e:
73            print(s.error(e))
74            print(s.warn("Invalid method name."))
75            exit()
76        
77        result = method(**kwargs)
78        if isinstance(result, dict):
79            print(json.dumps(result, indent=2))
80        else:
81            print(result)
s = <arches_extensions.utils.ArchesCLIStyles object>
class Command(django.core.management.base.BaseCommand): 
11class Command(BaseCommand):
12    """
13    Run ETL modules from the command line instead of through a browser.
14
15    Usage:
16
17        python manage.py etl [module_name] [method_name] [--opts arg1=val1 arg2=val2]
18
19    A rudimentary and generic implementation so far, allows you to run a specific method on a ETL module, passing in keyword arguments as needed. This can be used during the development of ETL modules, or later to run server-side ETL operations that mimic the functionality available through the browser.
20    """
21
22    def __init__(self, *args, **kwargs):
23        self.help = self.__doc__
24
25    def add_arguments(self, parser):
26        parser.formatter_class = ArchesHelpTextFormatter
27        parser.add_argument("module_name",
28            help=f"Name of ETL module to use. Use {s.fg.pink}python manage.py extension list etl-module{s.reset} to get a list of valid ETL module names. Remember to use quotes around names that contain spaces."
29        )
30        parser.add_argument("method_name",
31            help='Name of method on ETL module class to run.'
32        )
33        parser.add_argument("--opts",
34            nargs='*',
35            help=f"Pass arbitrary keyword arguments to the method. Use format {s.opt('--opts arg1=val1 arg2=val2')}"
36        )
37
38    def handle(self, *args, **options):
39
40        try:
41            opts = {}
42            for i in options['opts']:
43                k, v = i.split("=")
44                if v == "True":
45                    v = True
46                elif v == "False":
47                    v = False
48                opts[k] = v
49        except Exception as e:
50            print(s.error(options['opts']))
51            print(s.warn("Invalid opts list. Format must be --opts arg1=val1 arg2=val2"))
52            exit()
53
54        self.run_method(options['module_name'], options['method_name'], **opts)
55
56    def print_module_list(self):
57        ExtensionManager("etl-module").print_list()
58
59    def run_method(self, module_name, method_name, **kwargs):
60
61        try:
62            instance = ExtensionManager("etl-module")._get_instance(module_name)
63        except Exception as e:
64            print(s.error(e))
65            print(s.warn("Valid module names:"))
66            self.print_module_list()
67            exit()
68
69        module_class = instance.get_class_module()
70        module = module_class()
71        try:
72            method = getattr(module, method_name)
73        except AttributeError as e:
74            print(s.error(e))
75            print(s.warn("Invalid method name."))
76            exit()
77        
78        result = method(**kwargs)
79        if isinstance(result, dict):
80            print(json.dumps(result, indent=2))
81        else:
82            print(result)

Run ETL modules from the command line instead of through a browser.

Usage:

python manage.py etl [module_name] [method_name] [--opts arg1=val1 arg2=val2]

A rudimentary and generic implementation so far, allows you to run a specific method on a ETL module, passing in keyword arguments as needed. This can be used during the development of ETL modules, or later to run server-side ETL operations that mimic the functionality available through the browser.

Command(*args, **kwargs) 
22    def __init__(self, *args, **kwargs):
23        self.help = self.__doc__
help = ''
def add_arguments(self, parser): 
25    def add_arguments(self, parser):
26        parser.formatter_class = ArchesHelpTextFormatter
27        parser.add_argument("module_name",
28            help=f"Name of ETL module to use. Use {s.fg.pink}python manage.py extension list etl-module{s.reset} to get a list of valid ETL module names. Remember to use quotes around names that contain spaces."
29        )
30        parser.add_argument("method_name",
31            help='Name of method on ETL module class to run.'
32        )
33        parser.add_argument("--opts",
34            nargs='*',
35            help=f"Pass arbitrary keyword arguments to the method. Use format {s.opt('--opts arg1=val1 arg2=val2')}"
36        )

Entry point for subclassed commands to add custom arguments.

def handle(self, *args, **options): 
38    def handle(self, *args, **options):
39
40        try:
41            opts = {}
42            for i in options['opts']:
43                k, v = i.split("=")
44                if v == "True":
45                    v = True
46                elif v == "False":
47                    v = False
48                opts[k] = v
49        except Exception as e:
50            print(s.error(options['opts']))
51            print(s.warn("Invalid opts list. Format must be --opts arg1=val1 arg2=val2"))
52            exit()
53
54        self.run_method(options['module_name'], options['method_name'], **opts)

The actual logic of the command. Subclasses must implement this method.

def print_module_list(self): 
56    def print_module_list(self):
57        ExtensionManager("etl-module").print_list()
def run_method(self, module_name, method_name, **kwargs): 
59    def run_method(self, module_name, method_name, **kwargs):
60
61        try:
62            instance = ExtensionManager("etl-module")._get_instance(module_name)
63        except Exception as e:
64            print(s.error(e))
65            print(s.warn("Valid module names:"))
66            self.print_module_list()
67            exit()
68
69        module_class = instance.get_class_module()
70        module = module_class()
71        try:
72            method = getattr(module, method_name)
73        except AttributeError as e:
74            print(s.error(e))
75            print(s.warn("Invalid method name."))
76            exit()
77        
78        result = method(**kwargs)
79        if isinstance(result, dict):
80            print(json.dumps(result, indent=2))
81        else:
82            print(result)