GNOME.org
 

[Notes] [Git][BuildStream/buildstream][bschubert/loader] 2 commits: Add LoadElement in dependency list for LoadElement idrectly

Title: GitLab

Benjamin Schubert pushed to branch bschubert/loader at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/_loader/loadelement.py
    ... ... @@ -39,6 +39,10 @@ from .types import Symbol, Dependency
    39 39 
     #    loader (Loader): The Loader object for this element
    40 40 
     #
    41 41 
     class LoadElement():
    42 
    +    class Dependency:
    43 
    +        def __init__(self, element, dep_type):
    44 
    +            self.element = element
    45 
    +            self.dep_type = dep_type
    42 46
    43 47 
         def __init__(self, node, filename, loader):
    44 48
    ... ... @@ -74,8 +78,11 @@ class LoadElement():
    74 78 
                 'build-depends', 'runtime-depends',
    75 79 
             ])
    76 80
    77 
    -        # Extract the Dependencies
    78 
    -        self.deps = _extract_depends_from_node(self.node)
    81 
    +        self.dependencies = []
    82 
    +
    83 
    +    @property
    84 
    +    def junction(self):
    85 
    +        return self._loader.project.junction
    79 86
    80 87 
         # depends():
    81 88 
         #
    ... ... @@ -101,8 +108,8 @@ class LoadElement():
    101 108 
                 return
    102 109
    103 110 
             self._dep_cache = {}
    104 
    -        for dep in self.deps:
    105 
    -            elt = self._loader.get_element_for_dep(dep)
    111 
    +        for dep in self.dependencies:
    112 
    +            elt = dep.element
    106 113
    107 114 
                 # Ensure the cache of the element we depend on
    108 115 
                 elt._ensure_depends_cache()

  • buildstream/_loader/loader.py
    ... ... @@ -33,7 +33,7 @@ from .._includes import Includes
    33 33 
     from .._yamlcache import YamlCache
    34 34
    35 35 
     from .types import Symbol, Dependency
    36 
    -from .loadelement import LoadElement
    36 
    +from .loadelement import LoadElement, _extract_depends_from_node
    37 37 
     from . import MetaElement
    38 38 
     from . import MetaSource
    39 39 
     from ..types import CoreWarnings
    ... ... @@ -112,7 +112,7 @@ class Loader():
    112 112
    113 113 
             # First pass, recursively load files and populate our table of LoadElements
    114 114 
             #
    115 
    -        deps = []
    115 
    +        target_elements = []
    116 116
    117 117 
             # XXX This will need to be changed to the context's top-level project if this method
    118 118 
             # is ever used for subprojects
    ... ... @@ -124,8 +124,8 @@ class Loader():
    124 124 
                     profile_start(Topics.LOAD_PROJECT, target)
    125 125 
                     junction, name, loader = self._parse_name(target, rewritable, ticker,
    126 126 
                                                               fetch_subprojects=fetch_subprojects)
    127 
    -                loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    -                deps.append(Dependency(name, junction=junction))
    127 
    +                element = loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    +                target_elements.append(element)
    129 129 
                     profile_end(Topics.LOAD_PROJECT, target)
    130 130
    131 131 
             #
    ... ... @@ -134,29 +134,29 @@ class Loader():
    134 134
    135 135 
             # Set up a dummy element that depends on all top-level targets
    136 136 
             # to resolve potential circular dependencies between them
    137 
    -        DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
    138 
    -
    139 
    -        dummy = DummyTarget(name='', full_name='', deps=deps)
    140 
    -        self._elements[''] = dummy
    137 
    +        dummy_target = LoadElement("", "", self)
    138 
    +        dummy_target.dependencies.extend(
    139 
    +            LoadElement.Dependency(element, Symbol.RUNTIME)
    140 
    +            for element in target_elements
    141 
    +        )
    141 142
    142 143 
             profile_key = "_".join(t for t in targets)
    143 144 
             profile_start(Topics.CIRCULAR_CHECK, profile_key)
    144 
    -        self._check_circular_deps('')
    145 
    +        self._check_circular_deps(dummy_target)
    145 146 
             profile_end(Topics.CIRCULAR_CHECK, profile_key)
    146 147
    147 148 
             ret = []
    148 149 
             #
    149 150 
             # Sort direct dependencies of elements by their dependency ordering
    150 151 
             #
    151 
    -        for target in targets:
    152 
    -            profile_start(Topics.SORT_DEPENDENCIES, target)
    153 
    -            junction, name, loader = self._parse_name(target, rewritable, ticker,
    154 
    -                                                      fetch_subprojects=fetch_subprojects)
    155 
    -            loader._sort_dependencies(name)
    156 
    -            profile_end(Topics.SORT_DEPENDENCIES, target)
    152 
    +        for element in target_elements:
    153 
    +            loader = element._loader
    154 
    +            profile_start(Topics.SORT_DEPENDENCIES, element.name)
    155 
    +            loader._sort_dependencies(element)
    156 
    +            profile_end(Topics.SORT_DEPENDENCIES, element.name)
    157 157 
                 # Finally, wrap what we have into LoadElements and return the target
    158 158 
                 #
    159 
    -            ret.append(loader._collect_element(name))
    159 
    +            ret.append(loader._collect_element(element))
    160 160
    161 161 
             return ret
    162 162
    ... ... @@ -184,22 +184,6 @@ class Loader():
    184 184 
                 if os.path.exists(self._tempdir):
    185 185 
                     shutil.rmtree(self._tempdir)
    186 186
    187 
    -    # get_element_for_dep():
    188 
    -    #
    189 
    -    # Gets a cached LoadElement by Dependency object
    190 
    -    #
    191 
    -    # This is used by LoadElement
    192 
    -    #
    193 
    -    # Args:
    194 
    -    #    dep (Dependency): The dependency to search for
    195 
    -    #
    196 
    -    # Returns:
    197 
    -    #    (LoadElement): The cached LoadElement
    198 
    -    #
    199 
    -    def get_element_for_dep(self, dep):
    200 
    -        loader = self._get_loader_for_dep(dep)
    201 
    -        return loader._elements[dep.name]
    202 
    -
    203 187 
         ###########################################
    204 188 
         #            Private Methods              #
    205 189 
         ###########################################
    ... ... @@ -272,8 +256,10 @@ class Loader():
    272 256
    273 257 
             self._elements[filename] = element
    274 258
    259 
    +        dependencies = _extract_depends_from_node(node)
    260 
    +
    275 261 
             # Load all dependency files for the new LoadElement
    276 
    -        for dep in element.deps:
    262 
    +        for dep in dependencies:
    277 263 
                 if dep.junction:
    278 264 
                     self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    279 265 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    ... ... @@ -288,7 +274,9 @@ class Loader():
    288 274 
                                     "{}: Cannot depend on junction"
    289 275 
                                     .format(dep.provenance))
    290 276
    291 
    -        deps_names = [dep.name for dep in element.deps]
    277 
    +            element.dependencies.append(LoadElement.Dependency(dep_element, dep.dep_type))
    278 
    +
    279 
    +        deps_names = [dep.name for dep in dependencies]
    292 280 
             self._warn_invalid_elements(deps_names)
    293 281
    294 282 
             return element
    ... ... @@ -299,12 +287,12 @@ class Loader():
    299 287 
         # dependencies already resolved.
    300 288 
         #
    301 289 
         # Args:
    302 
    -    #    element_name (str): The element-path relative element name to check
    290 
    +    #    element (str): The element to check
    303 291 
         #
    304 292 
         # Raises:
    305 293 
         #    (LoadError): In case there was a circular dependency error
    306 294 
         #
    307 
    -    def _check_circular_deps(self, element_name, check_elements=None, validated=None, sequence=None):
    295 
    +    def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
    308 296
    309 297 
             if check_elements is None:
    310 298 
                 check_elements = {}
    ... ... @@ -313,38 +301,31 @@ class Loader():
    313 301 
             if sequence is None:
    314 302 
                 sequence = []
    315 303
    316 
    -        element = self._elements[element_name]
    317 
    -
    318 
    -        # element name must be unique across projects
    319 
    -        # to be usable as key for the check_elements and validated dicts
    320 
    -        element_name = element.full_name
    321 
    -
    322 304 
             # Skip already validated branches
    323 
    -        if validated.get(element_name) is not None:
    305 
    +        if validated.get(element) is not None:
    324 306 
                 return
    325 307
    326 
    -        if check_elements.get(element_name) is not None:
    308 
    +        if check_elements.get(element) is not None:
    327 309 
                 # Create `chain`, the loop of element dependencies from this
    328 310 
                 # element back to itself, by trimming everything before this
    329 311 
                 # element from the sequence under consideration.
    330 
    -            chain = sequence[sequence.index(element_name):]
    331 
    -            chain.append(element_name)
    312 
    +            chain = sequence[sequence.index(element.full_name):]
    313 
    +            chain.append(element.full_name)
    332 314 
                 raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
    333 315 
                                 ("Circular dependency detected at element: {}\n" +
    334 316 
                                  "Dependency chain: {}")
    335 
    -                            .format(element.name, " -> ".join(chain)))
    317 
    +                            .format(element.full_name, " -> ".join(chain)))
    336 318
    337 319 
             # Push / Check each dependency / Pop
    338 
    -        check_elements[element_name] = True
    339 
    -        sequence.append(element_name)
    340 
    -        for dep in element.deps:
    341 
    -            loader = self._get_loader_for_dep(dep)
    342 
    -            loader._check_circular_deps(dep.name, check_elements, validated, sequence)
    343 
    -        del check_elements[element_name]
    320 
    +        check_elements[element] = True
    321 
    +        sequence.append(element.full_name)
    322 
    +        for dep in element.dependencies:
    323 
    +            dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
    324 
    +        del check_elements[element]
    344 325 
             sequence.pop()
    345 326
    346 327 
             # Eliminate duplicate paths
    347 
    -        validated[element_name] = True
    328 
    +        validated[element] = True
    348 329
    349 330 
         # _sort_dependencies():
    350 331 
         #
    ... ... @@ -357,28 +338,21 @@ class Loader():
    357 338 
         # sorts throughout the build process.
    358 339 
         #
    359 340 
         # Args:
    360 
    -    #    element_name (str): The element-path relative element name to sort
    341 
    +    #    element (LoadElement): The element to sort
    361 342 
         #
    362 
    -    def _sort_dependencies(self, element_name, visited=None):
    343 
    +    def _sort_dependencies(self, element, visited=None):
    363 344 
             if visited is None:
    364 
    -            visited = {}
    345 
    +            visited = set()
    365 346
    366 
    -        element = self._elements[element_name]
    367 
    -
    368 
    -        # element name must be unique across projects
    369 
    -        # to be usable as key for the visited dict
    370 
    -        element_name = element.full_name
    371 
    -
    372 
    -        if visited.get(element_name) is not None:
    347 
    +        if element in visited:
    373 348 
                 return
    374 349
    375 
    -        for dep in element.deps:
    376 
    -            loader = self._get_loader_for_dep(dep)
    377 
    -            loader._sort_dependencies(dep.name, visited=visited)
    350 
    +        for dep in element.dependencies:
    351 
    +            dep.element._loader._sort_dependencies(dep.element, visited=visited)
    378 352
    379 353 
             def dependency_cmp(dep_a, dep_b):
    380 
    -            element_a = self.get_element_for_dep(dep_a)
    381 
    -            element_b = self.get_element_for_dep(dep_b)
    354 
    +            element_a = dep_a.element
    355 
    +            element_b = dep_b.element
    382 356
    383 357 
                 # Sort on inter element dependency first
    384 358 
                 if element_a.depends(element_b):
    ... ... @@ -395,21 +369,21 @@ class Loader():
    395 369 
                         return -1
    396 370
    397 371 
                 # All things being equal, string comparison.
    398 
    -            if dep_a.name > dep_b.name:
    372 
    +            if element_a.name > element_b.name:
    399 373 
                     return 1
    400 
    -            elif dep_a.name < dep_b.name:
    374 
    +            elif element_a.name < element_b.name:
    401 375 
                     return -1
    402 376
    403 377 
                 # Sort local elements before junction elements
    404 378 
                 # and use string comparison between junction elements
    405 
    -            if dep_a.junction and dep_b.junction:
    406 
    -                if dep_a.junction > dep_b.junction:
    379 
    +            if element_a.junction and element_b.junction:
    380 
    +                if element_a.junction > element_b.junction:
    407 381 
                         return 1
    408 
    -                elif dep_a.junction < dep_b.junction:
    382 
    +                elif element_a.junction < element_b.junction:
    409 383 
                         return -1
    410 
    -            elif dep_a.junction:
    384 
    +            elif element_a.junction:
    411 385 
                     return -1
    412 
    -            elif dep_b.junction:
    386 
    +            elif element_b.junction:
    413 387 
                     return 1
    414 388
    415 389 
                 # This wont ever happen
    ... ... @@ -418,26 +392,23 @@ class Loader():
    418 392 
             # Now dependency sort, we ensure that if any direct dependency
    419 393 
             # directly or indirectly depends on another direct dependency,
    420 394 
             # it is found later in the list.
    421 
    -        element.deps.sort(key=cmp_to_key(dependency_cmp))
    395 
    +        element.dependencies.sort(key=cmp_to_key(dependency_cmp))
    422 396
    423 
    -        visited[element_name] = True
    397 
    +        visited.add(element)
    424 398
    425 399 
         # _collect_element()
    426 400 
         #
    427 401 
         # Collect the toplevel elements we have
    428 402 
         #
    429 403 
         # Args:
    430 
    -    #    element_name (str): The element-path relative element name to sort
    404 
    +    #    element (LoadElement): The element for which to load a MetaElement
    431 405 
         #
    432 406 
         # Returns:
    433 407 
         #    (MetaElement): A recursively loaded MetaElement
    434 408 
         #
    435 
    -    def _collect_element(self, element_name):
    436 
    -
    437 
    -        element = self._elements[element_name]
    438 
    -
    409 
    +    def _collect_element(self, element):
    439 410 
             # Return the already built one, if we already built it
    440 
    -        meta_element = self._meta_elements.get(element_name)
    411 
    +        meta_element = self._meta_elements.get(element.name)
    441 412 
             if meta_element:
    442 413 
                 return meta_element
    443 414
    ... ... @@ -461,10 +432,10 @@ class Loader():
    461 432 
                     del source[Symbol.DIRECTORY]
    462 433
    463 434 
                 index = sources.index(source)
    464 
    -            meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
    435 
    +            meta_source = MetaSource(element.full_name, index, element_kind, kind, source, directory)
    465 436 
                 meta_sources.append(meta_source)
    466 437
    467 
    -        meta_element = MetaElement(self.project, element_name, element_kind,
    438 
    +        meta_element = MetaElement(self.project, element.name, element_kind,
    468 439 
                                        elt_provenance, meta_sources,
    469 440 
                                        _yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={}),
    470 441 
                                        _yaml.node_get(node, Mapping, Symbol.VARIABLES, default_value={}),
    ... ... @@ -475,12 +446,12 @@ class Loader():
    475 446 
                                        element_kind == 'junction')
    476 447
    477 448 
             # Cache it now, make sure it's already there before recursing
    478 
    -        self._meta_elements[element_name] = meta_element
    449 
    +        self._meta_elements[element.name] = meta_element
    479 450
    480 451 
             # Descend
    481 
    -        for dep in element.deps:
    482 
    -            loader = self._get_loader_for_dep(dep)
    483 
    -            meta_dep = loader._collect_element(dep.name)
    452 
    +        for dep in element.dependencies:
    453 
    +            loader = dep.element._loader
    454 
    +            meta_dep = loader._collect_element(dep.element)
    484 455 
                 if dep.dep_type != 'runtime':
    485 456 
                     meta_element.build_dependencies.append(meta_dep)
    486 457 
                 if dep.dep_type != 'build':
    ... ... @@ -539,7 +510,7 @@ class Loader():
    539 510 
                     return None
    540 511
    541 512 
             # meta junction element
    542 
    -        meta_element = self._collect_element(filename)
    513 
    +        meta_element = self._collect_element(self._elements[filename])
    543 514 
             if meta_element.kind != 'junction':
    544 515 
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    545 516 
                                 "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
    ... ... @@ -601,23 +572,6 @@ class Loader():
    601 572
    602 573 
             return loader
    603 574
    604 
    -    # _get_loader_for_dep():
    605 
    -    #
    606 
    -    # Gets the appropriate Loader for a Dependency object
    607 
    -    #
    608 
    -    # Args:
    609 
    -    #    dep (Dependency): A Dependency object
    610 
    -    #
    611 
    -    # Returns:
    612 
    -    #    (Loader): The Loader object to use for this Dependency
    613 
    -    #
    614 
    -    def _get_loader_for_dep(self, dep):
    615 
    -        if dep.junction:
    616 
    -            # junction dependency, delegate to appropriate loader
    617 
    -            return self._loaders[dep.junction]
    618 
    -        else:
    619 
    -            return self
    620 
    -
    621 575 
         # _parse_name():
    622 576 
         #
    623 577 
         # Get junction and base name of element along with loader for the sub-project

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]