#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Generic config parsing and dumping, the way I remember it from scripts
gone by.

The config should be built from script-level defaults, overlaid by
config-file defaults, overlaid by command line options.

  (For buildbot-analogues that would be factory-level defaults,
   builder-level defaults, and build request/scheduler settings.)

The config should then be locked (set to read-only, to prevent runtime
alterations).  Afterwards we should dump the config to a file that is
uploaded with the build, and can be used to debug or replicate the build
at a later time.

TODO:

* check_required_settings or something -- run at init, assert that
  these settings are set.
"""

from __future__ import print_function

import os
import socket
import sys
import time
from copy import deepcopy
from optparse import Option, OptionGroup, OptionParser

from mozharness.base.log import CRITICAL, DEBUG, ERROR, FATAL, INFO, WARNING

try:
    from urllib2 import URLError, urlopen
except ImportError:
    from urllib.request import urlopen
    from urllib.error import URLError


try:
    import simplejson as json
except ImportError:
    import json


# optparse {{{1
class ExtendedOptionParser(OptionParser):
    """OptionParser, but with ExtendOption as the option_class.
    """

    def __init__(self, **kwargs):
        kwargs['option_class'] = ExtendOption
        OptionParser.__init__(self, **kwargs)


class ExtendOption(Option):
    """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions"""
    ACTIONS = Option.ACTIONS + ("extend",)
    STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
    TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)
    ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",)

    def take_action(self, action, dest, opt, value, values, parser):
        if action == "extend":
            lvalue = value.split(",")
            values.ensure_value(dest, []).extend(lvalue)
        else:
            Option.take_action(
                self, action, dest, opt, value, values, parser)


def make_immutable(item):
    if isinstance(item, list) or isinstance(item, tuple):
        result = LockedTuple(item)
    elif isinstance(item, dict):
        result = ReadOnlyDict(item)
        result.lock()
    else:
        result = item
    return result


class LockedTuple(tuple):
    def __new__(cls, items):
        return tuple.__new__(cls, (make_immutable(x) for x in items))

    def __deepcopy__(self, memo):
        return [deepcopy(elem, memo) for elem in self]


# ReadOnlyDict {{{1
class ReadOnlyDict(dict):
    def __init__(self, dictionary):
        self._lock = False
        self.update(dictionary.copy())

    def _check_lock(self):
        assert not self._lock, "ReadOnlyDict is locked!"

    def lock(self):
        for (k, v) in list(self.items()):
            self[k] = make_immutable(v)
        self._lock = True

    def __setitem__(self, *args):
        self._check_lock()
        return dict.__setitem__(self, *args)

    def __delitem__(self, *args):
        self._check_lock()
        return dict.__delitem__(self, *args)

    def clear(self, *args):
        self._check_lock()
        return dict.clear(self, *args)

    def pop(self, *args):
        self._check_lock()
        return dict.pop(self, *args)

    def popitem(self, *args):
        self._check_lock()
        return dict.popitem(self, *args)

    def setdefault(self, *args):
        self._check_lock()
        return dict.setdefault(self, *args)

    def update(self, *args):
        self._check_lock()
        dict.update(self, *args)

    def __deepcopy__(self, memo):
        cls = self.__class__
        result = cls.__new__(cls)
        memo[id(self)] = result
        for k, v in list(self.__dict__.items()):
            setattr(result, k, deepcopy(v, memo))
        result._lock = False
        for k, v in list(self.items()):
            result[k] = deepcopy(v, memo)
        return result


DEFAULT_CONFIG_PATH = os.path.join(
    os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
    "configs",
)


# parse_config_file {{{1
def parse_config_file(file_name, quiet=False, search_path=None,
                      config_dict_name="config"):
    """Read a config file and return a dictionary.
    """
    file_path = None
    if os.path.exists(file_name):
        file_path = file_name
    else:
        if not search_path:
            search_path = ['.', DEFAULT_CONFIG_PATH]
        for path in search_path:
            if os.path.exists(os.path.join(path, file_name)):
                file_path = os.path.join(path, file_name)
                break
        else:
            raise IOError("Can't find %s in %s!" % (file_name, search_path))
    if file_name.endswith('.py'):
        global_dict = {}
        local_dict = {}
        exec(
            compile(
                open(
                    file_path,
                    "rb").read(),
                file_path,
                'exec'),
            global_dict,
            local_dict)
        config = local_dict[config_dict_name]
    elif file_name.endswith('.json'):
        fh = open(file_path)
        config = {}
        json_config = json.load(fh)
        config = dict(json_config)
        fh.close()
    else:
        raise RuntimeError(
            "Unknown config file type %s! (config files must end in .json or .py)" %
            file_name)
    # TODO return file_path
    return config


def download_config_file(url, file_name):
    n = 0
    attempts = 5
    sleeptime = 60
    max_sleeptime = 5 * 60
    while True:
        if n >= attempts:
            print(
                "Failed to download from url %s after %d attempts, quiting..." %
                (url, attempts))
            raise SystemError(-1)
        try:
            contents = urlopen(url, timeout=30).read()
            break
        except URLError as e:
            print("Error downloading from url %s: %s" % (url, str(e)))
        except socket.timeout as e:
            print("Time out accessing %s: %s" % (url, str(e)))
        except socket.error as e:
            print("Socket error when accessing %s: %s" % (url, str(e)))
        print("Sleeping %d seconds before retrying" % sleeptime)
        time.sleep(sleeptime)
        sleeptime = sleeptime * 2
        if sleeptime > max_sleeptime:
            sleeptime = max_sleeptime
        n += 1

    try:
        f = open(file_name, 'w')
        f.write(contents)
        f.close()
    except IOError as e:
        print(
            "Error writing downloaded contents to file %s: %s" %
            (file_name, str(e)))
        raise SystemError(-1)


# BaseConfig {{{1
class BaseConfig(object):
    """Basic config setting/getting.
    """

    def __init__(
            self,
            config=None,
            initial_config_file=None,
            config_options=None,
            all_actions=None,
            default_actions=None,
            volatile_config=None,
            option_args=None,
            require_config_file=False,
            append_env_variables_from_configs=False,
            usage="usage: %prog [options]"):
        self._config = {}
        self.all_cfg_files_and_dicts = []
        self.actions = []
        self.config_lock = False
        self.require_config_file = require_config_file
        # It allows to append env variables from multiple config files
        self.append_env_variables_from_configs = append_env_variables_from_configs

        if all_actions:
            self.all_actions = all_actions[:]
        else:
            self.all_actions = ['clobber', 'build']
        if default_actions:
            self.default_actions = default_actions[:]
        else:
            self.default_actions = self.all_actions[:]
        if volatile_config is None:
            self.volatile_config = {
                'actions': None,
                'add_actions': None,
                'no_actions': None,
            }
        else:
            self.volatile_config = deepcopy(volatile_config)

        if config:
            self.set_config(config)
        if initial_config_file:
            initial_config = parse_config_file(initial_config_file)
            self.all_cfg_files_and_dicts.append(
                (initial_config_file, initial_config)
            )
            self.set_config(initial_config)
            # Since initial_config_file is only set when running unit tests,
            # if no option_args have been specified, then the parser will
            # parse sys.argv which in this case would be the command line
            # options specified to run the tests, e.g. nosetests -v. Clearly,
            # the options passed to nosetests (such as -v) should not be
            # interpreted by mozharness as mozharness options, so we specify
            # a dummy command line with no options, so that the parser does
            # not add anything from the test invocation command line
            # arguments to the mozharness options.
            if option_args is None:
                option_args = [
                    'dummy_mozharness_script_with_no_command_line_options.py']
        if config_options is None:
            config_options = []
        self._create_config_parser(config_options, usage)
        # we allow manually passing of option args for things like nosetests
        self.parse_args(args=option_args)

    def get_read_only_config(self):
        return ReadOnlyDict(self._config)

    def _create_config_parser(self, config_options, usage):
        self.config_parser = ExtendedOptionParser(usage=usage)
        self.config_parser.add_option(
            "--work-dir", action="store", dest="work_dir",
            type="string", default="build",
            help="Specify the work_dir (subdir of base_work_dir)"
        )
        self.config_parser.add_option(
            "--base-work-dir",
            action="store",
            dest="base_work_dir",
            type="string",
            default=os.getcwd(),
            help="Specify the absolute path of the parent of the working directory")
        self.config_parser.add_option(
            "--extra-config-path",
            action='extend',
            dest="config_paths",
            type="string",
            help="Specify additional paths to search for config files.",
        )
        self.config_parser.add_option(
            "-c", "--config-file", "--cfg", action="extend",
            dest="config_files", default=[], type="string",
            help="Specify a config file; can be repeated",
        )
        self.config_parser.add_option(
            "-C",
            "--opt-config-file",
            "--opt-cfg",
            action="extend",
            dest="opt_config_files",
            type="string",
            default=[],
            help="Specify an optional config file, like --config-file but with no "
            "error if the file is missing; can be repeated")
        self.config_parser.add_option(
            "--dump-config", action="store_true",
            dest="dump_config",
            help="List and dump the config generated from this run to "
                 "a JSON file."
        )
        self.config_parser.add_option(
            "--dump-config-hierarchy", action="store_true",
            dest="dump_config_hierarchy",
            help="Like --dump-config but will list and dump which config "
                 "files were used making up the config and specify their own "
                 "keys/values that were not overwritten by another cfg -- "
                 "held the highest hierarchy."
        )
        self.config_parser.add_option(
            "--append-env-variables-from-configs", action="store_true",
            dest="append_env_variables_from_configs",
            help="Merge environment variables from config files."
        )

        # Logging
        log_option_group = OptionGroup(self.config_parser, "Logging")
        log_option_group.add_option(
            "--log-level", action="store",
            type="choice", dest="log_level", default=INFO,
            choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL],
            help="Set log level (debug|info|warning|error|critical|fatal)"
        )
        log_option_group.add_option(
            "-q", "--quiet", action="store_false", dest="log_to_console",
            default=True, help="Don't log to the console"
        )
        log_option_group.add_option(
            "--append-to-log", action="store_true",
            dest="append_to_log", default=False,
            help="Append to the log"
        )
        log_option_group.add_option(
            "--multi-log", action="store_const", const="multi",
            dest="log_type", help="Log using MultiFileLogger"
        )
        log_option_group.add_option(
            "--simple-log", action="store_const", const="simple",
            dest="log_type", help="Log using SimpleFileLogger"
        )
        self.config_parser.add_option_group(log_option_group)

        # Actions
        action_option_group = OptionGroup(
            self.config_parser, "Actions",
            "Use these options to list or enable/disable actions."
        )
        action_option_group.add_option(
            "--list-actions", action="store_true",
            dest="list_actions",
            help="List all available actions, then exit"
        )
        action_option_group.add_option(
            "--add-action", action="extend",
            dest="add_actions", metavar="ACTIONS",
            help="Add action %s to the list of actions" % self.all_actions
        )
        action_option_group.add_option(
            "--no-action", action="extend",
            dest="no_actions", metavar="ACTIONS",
            help="Don't perform action"
        )
        for action in self.all_actions:
            action_option_group.add_option(
                "--%s" % action, action="append_const",
                dest="actions", const=action,
                help="Add %s to the limited list of actions" % action
            )
            action_option_group.add_option(
                "--no-%s" % action, action="append_const",
                dest="no_actions", const=action,
                help="Remove %s from the list of actions to perform" % action
            )
        self.config_parser.add_option_group(action_option_group)
        # Child-specified options
        # TODO error checking for overlapping options
        if config_options:
            for option in config_options:
                self.config_parser.add_option(*option[0], **option[1])

        # Initial-config-specified options
        config_options = self._config.get('config_options', None)
        if config_options:
            for option in config_options:
                self.config_parser.add_option(*option[0], **option[1])

    def set_config(self, config, overwrite=False):
        """This is probably doable some other way."""
        if self._config and not overwrite:
            self._config.update(config)
        else:
            self._config = config
        return self._config

    def get_actions(self):
        return self.actions

    def verify_actions(self, action_list, quiet=False):
        for action in action_list:
            if action not in self.all_actions:
                if not quiet:
                    print("Invalid action %s not in %s!" % (action,
                                                            self.all_actions))
                raise SystemExit(-1)
        return action_list

    def verify_actions_order(self, action_list):
        try:
            indexes = [self.all_actions.index(elt) for elt in action_list]
            sorted_indexes = sorted(indexes)
            for i in range(len(indexes)):
                if indexes[i] != sorted_indexes[i]:
                    print(("Action %s comes in different order in %s\n" +
                           "than in %s") %
                          (action_list[i], action_list, self.all_actions))
                    raise SystemExit(-1)
        except ValueError as e:
            print("Invalid action found: " + str(e))
            raise SystemExit(-1)

    def list_actions(self):
        print("Actions available:")
        for a in self.all_actions:
            print("    " + ("*" if a in self.default_actions else " "), a)
        raise SystemExit(0)

    def get_cfgs_from_files(self, all_config_files, options):
        """Returns the configuration derived from the list of configuration
        files.  The result is represented as a list of `(filename,
        config_dict)` tuples; they will be combined with keys in later
        dictionaries taking precedence over earlier.

        `all_config_files` is all files specified with `--config-file` and
        `--opt-config-file`; `options` is the argparse options object giving
        access to any other command-line options.

        This function is also responsible for downloading any configuration
        files specified by URL.  It uses ``parse_config_file`` in this module
        to parse individual files.

        This method can be overridden in a subclass to add extra logic to the
        way that self.config is made up.  See
        `mozharness.mozilla.building.buildbase.BuildingConfig` for an example.
        """
        config_paths = options.config_paths or ['.']
        all_cfg_files_and_dicts = []
        for cf in all_config_files:
            try:
                if '://' in cf:  # config file is an url
                    file_name = os.path.basename(cf)
                    file_path = os.path.join(os.getcwd(), file_name)
                    download_config_file(cf, file_path)
                    all_cfg_files_and_dicts.append(
                        (file_path, parse_config_file(file_path, search_path=["."]))
                    )
                else:
                    all_cfg_files_and_dicts.append(
                        (cf, parse_config_file(
                            cf,
                            search_path=config_paths + [DEFAULT_CONFIG_PATH]
                        ))
                    )
            except Exception:
                if cf in options.opt_config_files:
                    print(
                        "WARNING: optional config file not found %s" % cf
                    )
                else:
                    raise

        if 'EXTRA_MOZHARNESS_CONFIG' in os.environ:
            env_config = json.loads(os.environ['EXTRA_MOZHARNESS_CONFIG'])
            all_cfg_files_and_dicts.append(
                ("[EXTRA_MOZHARENSS_CONFIG]", env_config))

        return all_cfg_files_and_dicts

    def parse_args(self, args=None):
        """Parse command line arguments in a generic way.
        Return the parser object after adding the basic options, so
        child objects can manipulate it.
        """
        self.command_line = ' '.join(sys.argv)
        if args is None:
            args = sys.argv[1:]
        (options, args) = self.config_parser.parse_args(args)

        defaults = self.config_parser.defaults.copy()

        if not options.config_files:
            if self.require_config_file:
                if options.list_actions:
                    self.list_actions()
                print("Required config file not set! (use --config-file option)")
                raise SystemExit(-1)

        # this is what get_cfgs_from_files returns. It will represent each
        # config file name and its assoctiated dict
        # eg ('builds/branch_specifics.py', {'foo': 'bar'})
        # let's store this to self for things like --interpret-config-files
        self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
            # append opt_config to allow them to overwrite previous configs
            options.config_files + options.opt_config_files, options=options
        ))
        config = {}
        if (self.append_env_variables_from_configs
                or options.append_env_variables_from_configs):
            # We only append values from various configs for the 'env' entry
            # For everything else we follow the standard behaviour
            for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
                for v in list(c_dict.keys()):
                    if v == 'env' and v in config:
                        config[v].update(c_dict[v])
                    else:
                        config[v] = c_dict[v]
        else:
            for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
                config.update(c_dict)
        # assign or update self._config depending on if it exists or not
        #    NOTE self._config will be passed to ReadOnlyConfig's init -- a
        #    dict subclass with immutable locking capabilities -- and serve
        #    as the keys/values that make up that instance. Ultimately,
        #    this becomes self.config during BaseScript's init
        self.set_config(config)

        for key in list(defaults.keys()):
            value = getattr(options, key)
            if value is None:
                continue
            # Don't override config_file defaults with config_parser defaults
            if key in defaults and value == defaults[key] and key in self._config:
                continue
            self._config[key] = value

        # The idea behind the volatile_config is we don't want to save this
        # info over multiple runs.  This defaults to the action-specific
        # config options, but can be anything.
        for key in list(self.volatile_config.keys()):
            if self._config.get(key) is not None:
                self.volatile_config[key] = self._config[key]
                del(self._config[key])

        self.update_actions()
        if options.list_actions:
            self.list_actions()

        # Keep? This is for saving the volatile config in the dump_config
        self._config['volatile_config'] = self.volatile_config

        self.options = options
        self.args = args
        return (self.options, self.args)

    def update_actions(self):
        """ Update actions after reading in config.

        Seems a little complex, but the logic goes:

        First, if default_actions is specified in the config, set our
        default actions even if the script specifies other default actions.

        Without any other action-specific options, run with default actions.

        If we specify --ACTION or --only-ACTION once or multiple times,
        we want to override the default_actions list with the one(s) we list.

        Otherwise, if we specify --add-action ACTION, we want to add an
        action to the list.

        Finally, if we specify --no-ACTION, remove that from the list of
        actions to perform.
        """
        if self._config.get('default_actions'):
            default_actions = self.verify_actions(
                self._config['default_actions'])
            self.default_actions = default_actions
        self.verify_actions_order(self.default_actions)
        self.actions = self.default_actions[:]
        if self.volatile_config['actions']:
            actions = self.verify_actions(self.volatile_config['actions'])
            self.actions = actions
        elif self.volatile_config['add_actions']:
            actions = self.verify_actions(self.volatile_config['add_actions'])
            self.actions.extend(actions)
        if self.volatile_config['no_actions']:
            actions = self.verify_actions(self.volatile_config['no_actions'])
            for action in actions:
                if action in self.actions:
                    self.actions.remove(action)


# __main__ {{{1
if __name__ == '__main__':
    pass
