GNOME.org
 

[Notes] [Git][BuildStream/buildstream][bschubert/optimize-sort] fixup! fixup! fixup! fixup! Refactor sorting

Title: GitLab

Benjamin Schubert pushed to branch bschubert/optimize-sort at BuildStream / buildstream

Commits:

3 changed files:

Changes:

  • buildstream/_cachekey.py
    ... ... @@ -38,5 +38,8 @@ from . import _yaml
    38 38 
     #
    39 39 
     def generate_key(value):
    40 40 
         ordered = _yaml.node_sanitize(value)
    41 
    +    with open("data", "w") as fp:
    42 
    +        import pprint
    43 
    +        fp.write(pprint.pformat(ordered))
    41 44 
         string = pickle.dumps(ordered)
    42 45 
         return hashlib.sha256(string).hexdigest()

  • buildstream/_loader/loadelement.py
    ... ... @@ -43,7 +43,7 @@ class LoadElement():
    43 43
    44 44 
         next_id = itertools.count()
    45 45
    46 
    -    def __init__(self, node, filename, loader):
    46 
    +    def __init__(self, node, filename, loader, build_dependencies, dependencies):
    47 47
    48 48 
             #
    49 49 
             # Public members
    ... ... @@ -51,8 +51,9 @@ class LoadElement():
    51 51 
             self.node = node       # The YAML node
    52 52 
             self.name = filename   # The element name
    53 53 
             self.full_name = None  # The element full name (with associated junction)
    54 
    -        self.deps = None       # The list of Dependency objects
    55 
    -        self.index = None
    54 
    +        self.build_dependencies = build_dependencies
    55 
    +        self.dependencies = dependencies
    56 
    +        self.index = next(self.next_id)
    56 57
    57 58 
             #
    58 59 
             # Private members
    ... ... @@ -70,16 +71,16 @@ class LoadElement():
    70 71 
                 # dependency is in top-level project
    71 72 
                 self.full_name = self.name
    72 73
    73 
    -        # Ensure the root node is valid
    74 
    -        _yaml.node_validate(self.node, [
    75 
    -            'kind', 'depends', 'sources', 'sandbox',
    76 
    -            'variables', 'environment', 'environment-nocache',
    77 
    -            'config', 'public', 'description',
    78 
    -            'build-depends', 'runtime-depends',
    79 
    -        ])
    74 
    +        # # Ensure the root node is valid
    75 
    +        # _yaml.node_validate(self.node, [
    76 
    +        #     'kind', 'depends', 'sources', 'sandbox',
    77 
    +        #     'variables', 'environment', 'environment-nocache',
    78 
    +        #     'config', 'public', 'description',
    79 
    +        #     'build-depends', 'runtime-depends',
    80 
    +        # ])
    80 81
    81 82 
             # Extract the Dependencies
    82 
    -        self.deps = _extract_depends_from_node(self.node)
    83 
    +        # self.deps = _extract_depends_from_node(self.node)
    83 84
    84 85 
         # depends():
    85 86 
         #

  • buildstream/_loader/loader.py
    ... ... @@ -33,7 +33,7 @@ from .._includes import Includes
    33 33 
     from .._yamlcache import YamlCache
    34 34
    35 35 
     from .types import Symbol, Dependency
    36 
    -from .loadelement import LoadElement
    36 
    +from .loadelement import LoadElement, _extract_depends_from_node
    37 37 
     from . import MetaElement
    38 38 
     from . import MetaSource
    39 39 
     from ..types import CoreWarnings
    ... ... @@ -124,8 +124,8 @@ class Loader():
    124 124 
                     profile_start(Topics.LOAD_PROJECT, target)
    125 125 
                     junction, name, loader = self._parse_name(target, rewritable, ticker,
    126 126 
                                                               fetch_subprojects=fetch_subprojects)
    127 
    -                loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    -                deps.append(Dependency(name, junction=junction))
    127 
    +                element = loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    +                deps.append(element)
    129 129 
                     profile_end(Topics.LOAD_PROJECT, target)
    130 130
    131 131 
             #
    ... ... @@ -134,9 +134,7 @@ class Loader():
    134 134
    135 135 
             # Set up a dummy element that depends on all top-level targets
    136 136 
             # to resolve potential circular dependencies between them
    137 
    -        DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
    138 
    -
    139 
    -        dummy = DummyTarget(name='', full_name='', deps=deps)
    137 
    +        dummy = LoadElement(node='', filename='', build_dependencies=[], dependencies=deps, loader=self)
    140 138 
             self._elements[''] = dummy
    141 139
    142 140 
             profile_key = "_".join(t for t in targets)
    ... ... @@ -268,9 +266,16 @@ class Loader():
    268 266
    269 267 
                 self._options.process_node(node)
    270 268
    271 
    -        element = LoadElement(node, filename, self)
    269 
    +        _yaml.node_validate(node, [
    270 
    +            'kind', 'depends', 'sources', 'sandbox',
    271 
    +            'variables', 'environment', 'environment-nocache',
    272 
    +            'config', 'public', 'description',
    273 
    +            'build-depends', 'runtime-depends',
    274 
    +        ])
    272 275
    273 
    -        self._elements[filename] = element
    276 
    +        deps = _extract_depends_from_node(node)
    277 
    +        build_dependencies = []
    278 
    +        dependencies = []
    274 279
    275 280 
             def sort_deps(dep_a, dep_b):
    276 281 
                 if dep_a.dep_type != dep_b.dep_type:
    ... ... @@ -296,11 +301,7 @@ class Loader():
    296 301
    297 302 
                 raise SystemExit("Can't order stuff")
    298 303
    299 
    -        # Load all dependency files for the new LoadElement
    300 
    -        ### FIXME: HERE WE CODE
    301 
    -        ###
    302 
    -        ###
    303 
    -        for dep in sorted(element.deps, key=cmp_to_key(sort_deps)):
    304 
    +        for dep in sorted(deps, key=cmp_to_key(sort_deps)):
    304 305 
                 if dep.junction:
    305 306 
                     self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    306 307 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    ... ... @@ -310,20 +311,63 @@ class Loader():
    310 311
    311 312 
                 dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects, yaml_cache)
    312 313
    314 
    +            if dep.dep_type != "runtime":
    315 
    +                build_dependencies.append(dep_element)
    316 
    +            if dep.dep_type != "build":
    317 
    +                dependencies.append(dep_element)
    318 
    +
    313 319 
                 if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
    314 320 
                     raise LoadError(LoadErrorReason.INVALID_DATA,
    315 321 
                                     "{}: Cannot depend on junction"
    316 322 
                                     .format(dep.provenance))
    317 323
    318 
    -        deps_names = [dep.name for dep in element.deps]
    324 
    +        deps_names = [dep.name for dep in deps]
    319 325 
             self._warn_invalid_elements(deps_names)
    320 326
    321 
    -        element.deps.sort(key=cmp_to_key(sort_deps))
    327 
    +        def sort_by_index(a, b):
    328 
    +            return a.index - b.index
    322 329
    323 
    -        element.index = next(element.next_id)
    330 
    +        element = LoadElement(
    331 
    +            node,
    332 
    +            filename,
    333 
    +            self,
    334 
    +            list(sorted(build_dependencies, key=cmp_to_key(sort_by_index))),
    335 
    +            list(sorted(dependencies, key=cmp_to_key(sort_by_index))),
    336 
    +        )
    337 
    +
    338 
    +        self._elements[filename] = element
    324 339
    325 340 
             return element
    326 341
    342 
    +
    343 
    +        # # Load all dependency files for the new LoadElement
    344 
    +        # ### FIXME: HERE WE CODE
    345 
    +        # ###
    346 
    +        # ###
    347 
    +        # for dep in sorted(element.deps, key=cmp_to_key(sort_deps)):
    348 
    +        #     if dep.junction:
    349 
    +        #         self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    350 
    +        #         loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    351 
    +        #                                   fetch_subprojects=fetch_subprojects)
    352 
    +        #     else:
    353 
    +        #         loader = self
    354 
    +
    355 
    +        #     dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects, yaml_cache)
    356 
    +
    357 
    +        #     if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
    358 
    +        #         raise LoadError(LoadErrorReason.INVALID_DATA,
    359 
    +        #                         "{}: Cannot depend on junction"
    360 
    +        #                         .format(dep.provenance))
    361 
    +
    362 
    +        # deps_names = [dep.name for dep in element.deps]
    363 
    +        # self._warn_invalid_elements(deps_names)
    364 
    +
    365 
    +        # element.deps.sort(key=cmp_to_key(sort_deps))
    366 
    +
    367 
    +        # element.index = next(element.next_id)
    368 
    +
    369 
    +        # return element
    370 
    +
    327 371 
         # _check_circular_deps():
    328 372 
         #
    329 373 
         # Detect circular dependencies on LoadElements with
    ... ... @@ -368,7 +412,7 @@ class Loader():
    368 412 
             # Push / Check each dependency / Pop
    369 413 
             check_elements[element_name] = True
    370 414 
             sequence.append(element_name)
    371 
    -        for dep in element.deps:
    415 
    +        for dep in element.dependencies:
    372 416 
                 loader = self._get_loader_for_dep(dep)
    373 417 
                 loader._check_circular_deps(dep.name, check_elements, validated, sequence)
    374 418 
             del check_elements[element_name]
    ... ... @@ -509,13 +553,15 @@ class Loader():
    509 553 
             self._meta_elements[element_name] = meta_element
    510 554
    511 555 
             # Descend
    512 
    -        for dep in element.deps:
    513 
    -            loader = self._get_loader_for_dep(dep)
    514 
    -            meta_dep = loader._collect_element(dep.name)
    515 
    -            if dep.dep_type != 'runtime':
    516 
    -                meta_element.build_dependencies.append(meta_dep)
    517 
    -            if dep.dep_type != 'build':
    518 
    -                meta_element.dependencies.append(meta_dep)
    556 
    +        meta_element.build_dependencies = [
    557 
    +            self._get_loader_for_dep(dep)._collect_element(dep.name)
    558 
    +            for dep in element.build_dependencies
    559 
    +        ]
    560 
    +
    561 
    +        meta_element.dependencies = [
    562 
    +            self._get_loader_for_dep(dep)._collect_element(dep.name)
    563 
    +            for dep in element.dependencies
    564 
    +        ]
    519 565
    520 566 
             return meta_element
    521 567
    ... ... @@ -643,11 +689,7 @@ class Loader():
    643 689 
         #    (Loader): The Loader object to use for this Dependency
    644 690 
         #
    645 691 
         def _get_loader_for_dep(self, dep):
    646 
    -        if dep.junction:
    647 
    -            # junction dependency, delegate to appropriate loader
    648 
    -            return self._loaders[dep.junction]
    649 
    -        else:
    650 
    -            return self
    692 
    +        return dep._loader
    651 693
    652 694 
         # _parse_name():
    653 695 
         #

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]