GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/sysroot_dependencies] Add support for importing dependencies in sysroots.

Title: GitLab

Valentin David pushed to branch valentindavid/sysroot_dependencies at BuildStream / buildstream

Commits:

29 changed files:

Changes:

  • buildstream/_loader/loadelement.py
    ... ... @@ -72,10 +72,24 @@ class LoadElement():
    72 72 
                 'variables', 'environment', 'environment-nocache',
    73 73 
                 'config', 'public', 'description',
    74 74 
                 'build-depends', 'runtime-depends',
    75 
    +            'sysroots',
    75 76 
             ])
    76 77
    78 
    +        self.deps = []
    79 
    +        sysroots = _yaml.node_get(node, list, 'sysroots', default_value=[])
    80 
    +        for sysroot in sysroots:
    81 
    +            _yaml.node_validate(sysroot, ['path', 'depends', 'build-depends'])
    82 
    +            path = _yaml.node_get(sysroot, str, 'path')
    83 
    +            for dep in _extract_depends_from_node(sysroot):
    84 
    +                if dep.dep_type == Symbol.RUNTIME:
    85 
    +                    raise LoadError(LoadErrorReason.INVALID_DATA,
    86 
    +                                    "{}: Sysroot'ed dependencies can not be of type 'runtime'"
    87 
    +                                    .format(dep.provenance))
    88 
    +                self.deps.append((path, dep))
    89 
    +
    77 90 
             # Extract the Dependencies
    78 
    -        self.deps = _extract_depends_from_node(self.node)
    91 
    +        for dep in _extract_depends_from_node(self.node):
    92 
    +            self.deps.append(('/', dep))
    79 93
    80 94 
         # depends():
    81 95 
         #
    ... ... @@ -101,7 +115,7 @@ class LoadElement():
    101 115 
                 return
    102 116
    103 117 
             self._dep_cache = {}
    104 
    -        for dep in self.deps:
    118 
    +        for _, dep in self.deps:
    105 119 
                 elt = self._loader.get_element_for_dep(dep)
    106 120
    107 121 
                 # Ensure the cache of the element we depend on

  • buildstream/_loader/loader.py
    ... ... @@ -125,7 +125,7 @@ class Loader():
    125 125 
                     junction, name, loader = self._parse_name(target, rewritable, ticker,
    126 126 
                                                               fetch_subprojects=fetch_subprojects)
    127 127 
                     loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    -                deps.append(Dependency(name, junction=junction))
    128 
    +                deps.append(('/', Dependency(name, junction=junction)))
    129 129 
                     profile_end(Topics.LOAD_PROJECT, target)
    130 130
    131 131 
             #
    ... ... @@ -273,7 +273,7 @@ class Loader():
    273 273 
             self._elements[filename] = element
    274 274
    275 275 
             # Load all dependency files for the new LoadElement
    276 
    -        for dep in element.deps:
    276 
    +        for _, dep in element.deps:
    277 277 
                 if dep.junction:
    278 278 
                     self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    279 279 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    ... ... @@ -288,7 +288,7 @@ class Loader():
    288 288 
                                     "{}: Cannot depend on junction"
    289 289 
                                     .format(dep.provenance))
    290 290
    291 
    -        deps_names = [dep.name for dep in element.deps]
    291 
    +        deps_names = [dep.name for _, dep in element.deps]
    292 292 
             self._warn_invalid_elements(deps_names)
    293 293
    294 294 
             return element
    ... ... @@ -337,7 +337,7 @@ class Loader():
    337 337 
             # Push / Check each dependency / Pop
    338 338 
             check_elements[element_name] = True
    339 339 
             sequence.append(element_name)
    340 
    -        for dep in element.deps:
    340 
    +        for _, dep in element.deps:
    341 341 
                 loader = self._get_loader_for_dep(dep)
    342 342 
                 loader._check_circular_deps(dep.name, check_elements, validated, sequence)
    343 343 
             del check_elements[element_name]
    ... ... @@ -372,14 +372,21 @@ class Loader():
    372 372 
             if visited.get(element_name) is not None:
    373 373 
                 return
    374 374
    375 
    -        for dep in element.deps:
    375 
    +        for _, dep in element.deps:
    376 376 
                 loader = self._get_loader_for_dep(dep)
    377 377 
                 loader._sort_dependencies(dep.name, visited=visited)
    378 378
    379 
    -        def dependency_cmp(dep_a, dep_b):
    379 
    +        def dependency_cmp(sdep_a, sdep_b):
    380 
    +            sysroot_a, dep_a = sdep_a
    381 
    +            sysroot_b, dep_b = sdep_b
    380 382 
                 element_a = self.get_element_for_dep(dep_a)
    381 383 
                 element_b = self.get_element_for_dep(dep_b)
    382 384
    385 
    +            if sysroot_a < sysroot_b:
    386 
    +                return -1
    387 
    +            if sysroot_b < sysroot_a:
    388 
    +                return 1
    389 
    +
    383 390 
                 # Sort on inter element dependency first
    384 391 
                 if element_a.depends(element_b):
    385 392 
                     return 1
    ... ... @@ -478,11 +485,11 @@ class Loader():
    478 485 
             self._meta_elements[element_name] = meta_element
    479 486
    480 487 
             # Descend
    481 
    -        for dep in element.deps:
    488 
    +        for sysroot, dep in element.deps:
    482 489 
                 loader = self._get_loader_for_dep(dep)
    483 490 
                 meta_dep = loader._collect_element(dep.name)
    484 491 
                 if dep.dep_type != 'runtime':
    485 
    -                meta_element.build_dependencies.append(meta_dep)
    492 
    +                meta_element.build_dependencies.append((sysroot, meta_dep))
    486 493 
                 if dep.dep_type != 'build':
    487 494 
                     meta_element.dependencies.append(meta_dep)
    488 495

  • buildstream/buildelement.py
    ... ... @@ -224,8 +224,9 @@ class BuildElement(Element):
    224 224 
             # Run any integration commands provided by the dependencies
    225 225 
             # once they are all staged and ready
    226 226 
             with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
    227 
    -            for dep in self.dependencies(Scope.BUILD):
    228 
    -                dep.integrate(sandbox)
    227 
    +            for sysroot, dep in self.dependencies(Scope.BUILD, with_sysroot=True):
    228 
    +                if sysroot == '/':
    229 
    +                    dep.integrate(sandbox)
    229 230
    230 231 
             # Stage sources in the build root
    231 232 
             self.stage_sources(sandbox, self.get_variable('build-root'))

  • buildstream/element.py
    ... ... @@ -379,7 +379,8 @@ class Element(Plugin):
    379 379 
             for source in self.__sources:
    380 380 
                 yield source
    381 381
    382 
    -    def dependencies(self, scope, *, recurse=True, visited=None, recursed=False):
    382 
    +    def dependencies(self, scope, *, recurse=True, visited=None, recursed=False,
    383 
    +                     with_sysroot=False, sysroot='/'):
    383 384 
             """dependencies(scope, *, recurse=True)
    384 385
    385 386 
             A generator function which yields the dependencies of the given element.
    ... ... @@ -404,40 +405,56 @@ class Element(Plugin):
    404 405
    405 406 
             scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,))
    406 407
    407 
    -        if full_name in visited and scope_set.issubset(visited[full_name]):
    408 
    +        if with_sysroot:
    409 
    +            key = (sysroot, full_name)
    410 
    +        else:
    411 
    +            key = full_name
    412 
    +
    413 
    +        if key in visited and scope_set.issubset(visited[key]):
    408 414 
                 return
    409 415
    410 416 
             should_yield = False
    411 
    -        if full_name not in visited:
    412 
    -            visited[full_name] = scope_set
    417 
    +        if key not in visited:
    418 
    +            visited[key] = scope_set
    413 419 
                 should_yield = True
    414 420 
             else:
    415 
    -            visited[full_name] |= scope_set
    421 
    +            visited[key] |= scope_set
    416 422
    417 423 
             if recurse or not recursed:
    418 424 
                 if scope == Scope.ALL:
    419 
    -                for dep in self.__build_dependencies:
    425 
    +                build_deps = []
    426 
    +                for dep_sysroot, dep in self.__build_dependencies:
    427 
    +                    new_sysroot = self._subst_string(dep_sysroot) if not recursed else sysroot
    420 428 
                         yield from dep.dependencies(Scope.ALL, recurse=recurse,
    421 
    -                                                visited=visited, recursed=True)
    429 
    +                                                visited=visited, recursed=True,
    430 
    +                                                sysroot=new_sysroot, with_sysroot=with_sysroot)
    431 
    +                    build_deps.append(dep)
    422 432
    423 433 
                     for dep in self.__runtime_dependencies:
    424 
    -                    if dep not in self.__build_dependencies:
    434 
    +                    if dep not in build_deps:
    425 435 
                             yield from dep.dependencies(Scope.ALL, recurse=recurse,
    426 
    -                                                    visited=visited, recursed=True)
    436 
    +                                                    visited=visited, recursed=True,
    437 
    +                                                    sysroot=sysroot, with_sysroot=with_sysroot)
    427 438
    428 439 
                 elif scope == Scope.BUILD:
    429 
    -                for dep in self.__build_dependencies:
    440 
    +                for dep_sysroot, dep in self.__build_dependencies:
    441 
    +                    new_sysroot = self._subst_string(dep_sysroot) if not recursed else sysroot
    430 442 
                         yield from dep.dependencies(Scope.RUN, recurse=recurse,
    431 
    -                                                visited=visited, recursed=True)
    443 
    +                                                visited=visited, recursed=True,
    444 
    +                                                sysroot=new_sysroot, with_sysroot=with_sysroot)
    432 445
    433 446 
                 elif scope == Scope.RUN:
    434 447 
                     for dep in self.__runtime_dependencies:
    435 448 
                         yield from dep.dependencies(Scope.RUN, recurse=recurse,
    436 
    -                                                visited=visited, recursed=True)
    449 
    +                                                visited=visited, recursed=True,
    450 
    +                                                sysroot=sysroot, with_sysroot=with_sysroot)
    437 451
    438 452 
             # Yeild self only at the end, after anything needed has been traversed
    439 453 
             if should_yield and (recurse or recursed) and scope != Scope.BUILD:
    440 
    -            yield self
    454 
    +            if with_sysroot:
    455 
    +                yield sysroot, self
    456 
    +            else:
    457 
    +                yield self
    441 458
    442 459 
         def search(self, scope, name):
    443 460 
             """Search for a dependency by name
    ... ... @@ -636,7 +653,7 @@ class Element(Plugin):
    636 653 
                 vbasedir = sandbox.get_virtual_directory()
    637 654 
                 vstagedir = vbasedir \
    638 655 
                     if path is None \
    639 
    -                else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
    656 
    +                else vbasedir.descend(path.lstrip(os.sep).split(os.sep), create=True)
    640 657
    641 658 
                 files = list(self.__compute_splits(include, exclude, orphans))
    642 659
    ... ... @@ -654,7 +671,8 @@ class Element(Plugin):
    654 671 
             return link_result.combine(copy_result)
    655 672
    656 673 
         def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
    657 
    -                                   include=None, exclude=None, orphans=True):
    674 
    +                                   include=None, exclude=None, orphans=True,
    675 
    +                                   build=True):
    658 676 
             """Stage element dependencies in scope
    659 677
    660 678 
             This is primarily a convenience wrapper around
    ... ... @@ -669,6 +687,7 @@ class Element(Plugin):
    669 687 
                include (list): An optional list of domains to include files from
    670 688 
                exclude (list): An optional list of domains to exclude files from
    671 689 
                orphans (bool): Whether to include files not spoken for by split domains
    690 
    +           build (bool): Whether to stage artifacts for a build respecting sysroots
    672 691
    673 692 
             Raises:
    674 693 
                (:class:`.ElementError`): If any of the dependencies in `scope` have not
    ... ... @@ -684,7 +703,14 @@ class Element(Plugin):
    684 703 
             if self.__can_build_incrementally() and workspace.last_successful:
    685 704 
                 old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
    686 705
    687 
    -        for dep in self.dependencies(scope):
    706 
    +        def deps():
    707 
    +            if build:
    708 
    +                yield from self.dependencies(scope, with_sysroot=True)
    709 
    +            else:
    710 
    +                for dep in self.dependencies(scope, with_sysroot=False):
    711 
    +                    yield '/', dep
    712 
    +
    713 
    +        for sysroot, dep in deps():
    688 714 
                 # If we are workspaced, and we therefore perform an
    689 715 
                 # incremental build, we must ensure that we update the mtimes
    690 716 
                 # of any files created by our dependencies since the last
    ... ... @@ -709,8 +735,13 @@ class Element(Plugin):
    709 735 
                         if utils._is_main_process():
    710 736 
                             self._get_context().get_workspaces().save_config()
    711 737
    738 
    +            if build:
    739 
    +                sub_path = os.path.join(path, os.path.relpath(sysroot, '/')) if path else sysroot
    740 
    +            else:
    741 
    +                sub_path = path
    742 
    +
    712 743 
                 result = dep.stage_artifact(sandbox,
    713 
    -                                        path=path,
    744 
    +                                        path=sub_path,
    714 745 
                                             include=include,
    715 746 
                                             exclude=exclude,
    716 747 
                                             orphans=orphans,
    ... ... @@ -927,9 +958,9 @@ class Element(Plugin):
    927 958 
             for meta_dep in meta.dependencies:
    928 959 
                 dependency = Element._new_from_meta(meta_dep)
    929 960 
                 element.__runtime_dependencies.append(dependency)
    930 
    -        for meta_dep in meta.build_dependencies:
    961 
    +        for sysroot, meta_dep in meta.build_dependencies:
    931 962 
                 dependency = Element._new_from_meta(meta_dep)
    932 
    -            element.__build_dependencies.append(dependency)
    963 
    +            element.__build_dependencies.append((sysroot, dependency))
    933 964
    934 965 
             return element
    935 966
    ... ... @@ -1109,14 +1140,11 @@ class Element(Plugin):
    1109 1140 
                 # Weak cache key includes names of direct build dependencies
    1110 1141 
                 # but does not include keys of dependencies.
    1111 1142 
                 if self.BST_STRICT_REBUILD:
    1112 
    -                dependencies = [
    1113 
    -                    e._get_cache_key(strength=_KeyStrength.WEAK)
    1114 
    -                    for e in self.dependencies(Scope.BUILD)
    1115 
    -                ]
    1143 
    +                dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
    1144 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1116 1145 
                 else:
    1117 
    -                dependencies = [
    1118 
    -                    e.name for e in self.dependencies(Scope.BUILD, recurse=False)
    1119 
    -                ]
    1146 
    +                dependencies = [(sysroot, e.name)
    1147 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1120 1148
    1121 1149 
                 self.__weak_cache_key = self.__calculate_cache_key(dependencies)
    1122 1150
    ... ... @@ -1144,9 +1172,8 @@ class Element(Plugin):
    1144 1172 
                         return
    1145 1173
    1146 1174 
             if self.__strict_cache_key is None:
    1147 
    -            dependencies = [
    1148 
    -                e.__strict_cache_key for e in self.dependencies(Scope.BUILD)
    1149 
    -            ]
    1175 
    +            dependencies = [(sysroot, e.__strict_cache_key)
    1176 
    +                            for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1150 1177 
                 self.__strict_cache_key = self.__calculate_cache_key(dependencies)
    1151 1178
    1152 1179 
                 if self.__strict_cache_key is None:
    ... ... @@ -1186,10 +1213,8 @@ class Element(Plugin):
    1186 1213 
                     strong_key, _ = self.__get_artifact_metadata_keys()
    1187 1214 
                     self.__cache_key = strong_key
    1188 1215 
                 elif self.__assemble_scheduled or self.__assemble_done:
    1189 
    -                # Artifact will or has been built, not downloaded
    1190 
    -                dependencies = [
    1191 
    -                    e._get_cache_key() for e in self.dependencies(Scope.BUILD)
    1192 
    -                ]
    1216 
    +                dependencies = [(sysroot, e._get_cache_key())
    1217 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1193 1218 
                     self.__cache_key = self.__calculate_cache_key(dependencies)
    1194 1219
    1195 1220 
                 if self.__cache_key is None:
    ... ... @@ -1354,7 +1379,7 @@ class Element(Plugin):
    1354 1379 
                     else:
    1355 1380 
                         # Stage deps in the sandbox root
    1356 1381 
                         with self.timed_activity("Staging dependencies", silent_nested=True):
    1357 
    -                        self.stage_dependency_artifacts(sandbox, scope)
    1382 
    +                        self.stage_dependency_artifacts(sandbox, scope, build=False)
    1358 1383
    1359 1384 
                         # Run any integration commands provided by the dependencies
    1360 1385 
                         # once they are all staged and ready
    ... ... @@ -2080,8 +2105,11 @@ class Element(Plugin):
    2080 2105 
         #
    2081 2106 
         def __calculate_cache_key(self, dependencies):
    2082 2107 
             # No cache keys for dependencies which have no cache keys
    2083 
    -        if None in dependencies:
    2084 
    -            return None
    2108 
    +        for dep in dependencies:
    2109 
    +            if dep[1] is None:
    2110 
    +                return None
    2111 
    +        # Do not break cache keys
    2112 
    +        dependencies = [(sysroot, key) if sysroot != '/' else key for sysroot, key in dependencies]
    2085 2113
    2086 2114 
             # Generate dict that is used as base for all cache keys
    2087 2115 
             if self.__cache_key_dict is None:

  • buildstream/plugins/elements/compose.py
    ... ... @@ -123,8 +123,9 @@ class ComposeElement(Element):
    123 123 
                         vbasedir.mark_unmodified()
    124 124
    125 125 
                     with sandbox.batch(0):
    126 
    -                    for dep in self.dependencies(Scope.BUILD):
    127 
    -                        dep.integrate(sandbox)
    126 
    +                    for sysroot, dep in self.dependencies(Scope.BUILD, with_sysroot=True):
    127 
    +                        if sysroot == '/':
    128 
    +                            dep.integrate(sandbox)
    128 129
    129 130 
                     if require_split:
    130 131 
                         # Calculate added, modified and removed files

  • doc/source/format_declaring.rst
    ... ... @@ -159,6 +159,57 @@ See :ref:`format_dependencies` for more information on the dependency model.
    159 159
    160 160 
        The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
    161 161
    162 
    +Sysroot'ed dependencies
    163 
    +~~~~~~~~~~~~~~~~~~~~~~~
    164 
    +
    165 
    +Sysroot'ed dependencies are intended for bootstraping base systems or
    166 
    +cross-compiling.
    167 
    +
    168 
    +.. code:: yaml
    169 
    +
    170 
    +   # Specify some sysroot'ed dependencies
    171 
    +   sysroots:
    172 
    +   - path: /sysroot
    173 
    +     depends:
    174 
    +     - element1.bst
    175 
    +     - element2.bst
    176 
    +
    177 
    +During build, or initialization of build shell, sysroot'ed build
    178 
    +dependencies will be staged in the given sysroot path instead of '/'
    179 
    +together with the runtime dependencies of those sysroot'ed build
    180 
    +dependencies.
    181 
    +
    182 
    +It is possible to end up with indirect runtime dependencies in
    183 
    +different sysroots if they are staged from build dependencies with
    184 
    +different sysroots. They will be staged multiple times.
    185 
    +
    186 
    +Sysroot paths only apply to build dependencies. It is not possible to
    187 
    +define runtime dependencies either with ``type: runtime`` or
    188 
    +``runtime-depends``. It is possible to use ``all`` dependencies, but
    189 
    +the sysroot part is only for the build part not the runtime.
    190 
    +
    191 
    +For example:
    192 
    +
    193 
    +.. code:: yaml
    194 
    +
    195 
    +   sysroots:
    196 
    +   - path: /sysroot
    197 
    +     depends:
    198 
    +     - element.bst
    199 
    +
    200 
    +is equivalent to:
    201 
    +
    202 
    +.. code:: yaml
    203 
    +
    204 
    +   runtime-depends:
    205 
    +   - element.bst
    206 
    +   sysroots:
    207 
    +   - path: /sysroot
    208 
    +     build-depends:
    209 
    +     - element.bst
    210 
    +
    211 
    +:ref:`Integration commands <public_integration>` are never executed for
    212 
    +sysroot'ed dependencies.
    162 213
    163 214 
     .. _format_sources:
    164 215

  • tests/loader/dependencies.py
    ... ... @@ -135,7 +135,7 @@ def test_build_dependency(datafiles):
    135 135
    136 136 
         assert(len(element.build_dependencies) == 1)
    137 137 
         firstdep = element.build_dependencies[0]
    138 
    -    assert(isinstance(firstdep, MetaElement))
    138 
    +    assert(isinstance(firstdep[1], MetaElement))
    139 139
    140 140 
         assert(len(element.dependencies) == 0)
    141 141
    ... ... @@ -170,7 +170,7 @@ def test_build_runtime_dependency(datafiles):
    170 170 
         firstdep = element.dependencies[0]
    171 171 
         assert(isinstance(firstdep, MetaElement))
    172 172 
         firstbuilddep = element.build_dependencies[0]
    173 
    -    assert(firstdep == firstbuilddep)
    173 
    +    assert(firstdep == firstbuilddep[1])
    174 174
    175 175
    176 176 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -187,7 +187,7 @@ def test_all_dependency(datafiles):
    187 187 
         firstdep = element.dependencies[0]
    188 188 
         assert(isinstance(firstdep, MetaElement))
    189 189 
         firstbuilddep = element.build_dependencies[0]
    190 
    -    assert(firstdep == firstbuilddep)
    190 
    +    assert(firstdep == firstbuilddep[1])
    191 191
    192 192
    193 193 
     @pytest.mark.datafiles(DATA_DIR)

  • tests/sysroot_depends/project/elements/a.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/a

  • tests/sysroot_depends/project/elements/b.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/b

  • tests/sysroot_depends/project/elements/base.bst
    1 
    +kind: stack
    2 
    +depends:
    3 
    +- base/base-alpine.bst

  • tests/sysroot_depends/project/elements/base/base-alpine.bst
    1 
    +kind: import
    2 
    +
    3 
    +description: |
    4 
    +  Alpine Linux base for tests
    5 
    +
    6 
    +  Generated using the `tests/integration-tests/base/generate-base.sh` script.
    7 
    +
    8 
    +sources:
    9 
    +  - kind: tar
    10 
    +    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    11 
    +    base-dir: ''
    12 
    +    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

  • tests/sysroot_depends/project/elements/compose-integration.bst
    1 
    +kind: compose
    2 
    +
    3 
    +sysroots:
    4 
    +- path: /sysroot
    5 
    +  build-depends:
    6 
    +  - integration.bst

  • tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst
    1 
    +kind: compose
    2 
    +
    3 
    +sysroots:
    4 
    +- path: /other-sysroot
    5 
    +  build-depends:
    6 
    +  - layer2.bst

  • tests/sysroot_depends/project/elements/compose-layers.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- layer2.bst

  • tests/sysroot_depends/project/elements/integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +config:
    7 
    +  install-commands:
    8 
    +    - echo 0 >"%{install-root}/integrated.txt"
    9 
    +
    10 
    +public:
    11 
    +  bst:
    12 
    +    integration-commands:
    13 
    +    - echo 1 >/integrated.txt

  • tests/sysroot_depends/project/elements/layer1-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer1

  • tests/sysroot_depends/project/elements/layer1.bst
    1 
    +kind: stack
    2 
    +
    3 
    +depends:
    4 
    +- layer1-files.bst

  • tests/sysroot_depends/project/elements/layer2-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer2

  • tests/sysroot_depends/project/elements/layer2.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- layer2-files.bst
    5 
    +
    6 
    +build-depends:
    7 
    +- base.bst
    8 
    +
    9 
    +sysroots:
    10 
    +- path: /sysroot
    11 
    +  depends:
    12 
    +  - layer1.bst
    13 
    +
    14 
    +config:
    15 
    +  install-commands:
    16 
    +  - mkdir -p "%{install-root}"
    17 
    +  - |
    18 
    +    for file in /*; do
    19 
    +      if test -f "${file}"; then
    20 
    +        cp "${file}" "%{install-root}"
    21 
    +      fi
    22 
    +    done

  • tests/sysroot_depends/project/elements/manual-integration-runtime.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  depends:
    9 
    +  - integration.bst
    10 
    +
    11 
    +config:
    12 
    +  install-commands:
    13 
    +    - mkdir -p "%{install-root}"
    14 
    +    - echo dummy >"%{install-root}/dummy.txt"

  • tests/sysroot_depends/project/elements/manual-integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  build-depends:
    9 
    +  - integration.bst
    10 
    +
    11 
    +config:
    12 
    +  install-commands:
    13 
    +    - mkdir -p "%{install-root}/sysroot"
    14 
    +    - if test -f /sysroot/integrated.txt; then cp /sysroot/integrated.txt "%{install-root}/sysroot"; fi
    15 
    +    - if test -f /integrated.txt; then cp /integrated.txt "%{install-root}"; fi

  • tests/sysroot_depends/project/elements/target-variable.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- a.bst
    5 
    +
    6 
    +variables:
    7 
    +  mydir: test
    8 
    +
    9 
    +sysroots:
    10 
    +- path: "/path/%{mydir}"
    11 
    +  build-depends:
    12 
    +  - b.bst

  • tests/sysroot_depends/project/elements/target.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- a.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  build-depends:
    9 
    +  - b.bst

  • tests/sysroot_depends/project/files/a/a.txt
    1 
    +test

  • tests/sysroot_depends/project/files/b/b.txt
    1 
    +test

  • tests/sysroot_depends/project/files/layer1/1
    1 
    +1

  • tests/sysroot_depends/project/files/layer2/2
    1 
    +2

  • tests/sysroot_depends/project/project.conf
    1 
    +name: test
    2 
    +element-path: elements
    3 
    +aliases:
    4 
    +  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
    5 
    +options:
    6 
    +  linux:
    7 
    +    type: bool
    8 
    +    description: Whether to expect a linux platform
    9 
    +    default: True

  • tests/sysroot_depends/sysroot_depends.py
    1 
    +import os
    2 
    +import pytest
    3 
    +from tests.testutils import cli_integration as cli
    4 
    +from tests.testutils.site import IS_LINUX, HAVE_BWRAP
    5 
    +
    6 
    +
    7 
    +# Project directory
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project",
    11 
    +)
    12 
    +
    13 
    +
    14 
    +@pytest.mark.datafiles(DATA_DIR)
    15 
    +def test_sysroot_dependency_smoke_test(datafiles, cli, tmpdir):
    16 
    +    "Test simple sysroot use case without integration"
    17 
    +
    18 
    +    project = str(datafiles)
    19 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    20 
    +
    21 
    +    result = cli.run(project=project,
    22 
    +                     args=['build', 'target.bst'])
    23 
    +    result.assert_success()
    24 
    +
    25 
    +    result = cli.run(project=project,
    26 
    +                     args=['checkout', 'target.bst', checkout])
    27 
    +    result.assert_success()
    28 
    +    assert os.path.exists(os.path.join(checkout, 'a.txt'))
    29 
    +    assert os.path.exists(os.path.join(checkout, 'sysroot', 'b.txt'))
    30 
    +
    31 
    +
    32 
    +@pytest.mark.integration
    33 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    34 
    +@pytest.mark.datafiles(DATA_DIR)
    35 
    +def test_skip_integration_commands_compose(datafiles, cli, tmpdir):
    36 
    +    "Integration commands are not run on sysroots"
    37 
    +
    38 
    +    project = str(datafiles)
    39 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    40 
    +
    41 
    +    result = cli.run(project=project,
    42 
    +                     args=['build', 'compose-integration.bst'])
    43 
    +    result.assert_success()
    44 
    +
    45 
    +    result = cli.run(project=project,
    46 
    +                     args=['checkout', 'compose-integration.bst', checkout])
    47 
    +    result.assert_success()
    48 
    +
    49 
    +    integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
    50 
    +    assert os.path.exists(integrated)
    51 
    +    with open(integrated, 'r') as f:
    52 
    +        assert f.read() == '0\n'
    53 
    +
    54 
    +
    55 
    +@pytest.mark.integration
    56 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    57 
    +@pytest.mark.datafiles(DATA_DIR)
    58 
    +def test_skip_integration_commands_build_element(datafiles, cli, tmpdir):
    59 
    +    "Integration commands are not run on sysroots"
    60 
    +
    61 
    +    project = str(datafiles)
    62 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    63 
    +
    64 
    +    result = cli.run(project=project,
    65 
    +                     args=['build', 'manual-integration.bst'])
    66 
    +    result.assert_success()
    67 
    +
    68 
    +    result = cli.run(project=project,
    69 
    +                     args=['checkout', 'manual-integration.bst', checkout])
    70 
    +    result.assert_success()
    71 
    +
    72 
    +    sysroot_integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
    73 
    +    integrated = os.path.join(checkout, 'integrated.txt')
    74 
    +    assert os.path.exists(sysroot_integrated)
    75 
    +    with open(sysroot_integrated, 'r') as f:
    76 
    +        assert f.read() == '0\n'
    77 
    +    # We need to make sure that integration command has not been run on / either.
    78 
    +    assert not os.path.exists(integrated)
    79 
    +
    80 
    +
    81 
    +@pytest.mark.integration
    82 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    83 
    +@pytest.mark.datafiles(DATA_DIR)
    84 
    +def test_sysroot_only_for_build(cli, tmpdir, datafiles):
    85 
    +    project = str(datafiles)
    86 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    87 
    +
    88 
    +    result = cli.run(project=project,
    89 
    +                     args=['build', 'compose-layers.bst'])
    90 
    +    result.assert_success()
    91 
    +
    92 
    +    result = cli.run(project=project,
    93 
    +                     args=['checkout', 'compose-layers.bst', checkout])
    94 
    +
    95 
    +    result.assert_success()
    96 
    +    assert os.path.exists(os.path.join(checkout, '1'))
    97 
    +    assert os.path.exists(os.path.join(checkout, '2'))
    98 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    99 
    +
    100 
    +
    101 
    +@pytest.mark.integration
    102 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    103 
    +@pytest.mark.datafiles(DATA_DIR)
    104 
    +def test_sysroot_only_for_build_with_sysroot(cli, tmpdir, datafiles):
    105 
    +    project = str(datafiles)
    106 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    107 
    +
    108 
    +    result = cli.run(project=project,
    109 
    +                     args=['build', 'compose-layers-with-sysroot.bst'])
    110 
    +    result.assert_success()
    111 
    +
    112 
    +    result = cli.run(project=project,
    113 
    +                     args=['checkout', 'compose-layers-with-sysroot.bst', checkout])
    114 
    +
    115 
    +    result.assert_success()
    116 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '1'))
    117 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '2'))
    118 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    119 
    +
    120 
    +
    121 
    +@pytest.mark.integration
    122 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    123 
    +@pytest.mark.datafiles(DATA_DIR)
    124 
    +def test_shell_no_sysroot(cli, tmpdir, datafiles):
    125 
    +    "bst shell does not have sysroots and dependencies are integrated"
    126 
    +
    127 
    +    project = str(datafiles)
    128 
    +
    129 
    +    result = cli.run(project=project,
    130 
    +                     args=['build', 'base.bst', 'manual-integration-runtime.bst'])
    131 
    +    result.assert_success()
    132 
    +
    133 
    +    result = cli.run(project=project,
    134 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'cat', '/integrated.txt'])
    135 
    +    result.assert_success()
    136 
    +    assert result.output == '1\n'
    137 
    +
    138 
    +    result = cli.run(project=project,
    139 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'ls', '/sysroot/integrated.txt'])
    140 
    +    assert result.exit_code != 0
    141 
    +    assert result.output == ''
    142 
    +
    143 
    +
    144 
    +@pytest.mark.integration
    145 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    146 
    +@pytest.mark.datafiles(DATA_DIR)
    147 
    +def test_shell_build_sysroot(cli, tmpdir, datafiles):
    148 
    +    "Build shell should stage build dependencies sysroot'ed non integrated"
    149 
    +
    150 
    +    project = str(datafiles)
    151 
    +
    152 
    +    result = cli.run(project=project,
    153 
    +                     args=['build', 'base.bst', 'integration.bst'])
    154 
    +    result.assert_success()
    155 
    +
    156 
    +    result = cli.run(project=project,
    157 
    +                     args=['shell', '-b', 'manual-integration.bst', '--', 'cat', '/sysroot/integrated.txt'])
    158 
    +    result.assert_success()
    159 
    +    assert result.output == '0\n'
    160 
    +
    161 
    +
    162 
    +@pytest.mark.integration
    163 
    +@pytest.mark.datafiles(DATA_DIR)
    164 
    +def test_show_dependencies_only_once(cli, tmpdir, datafiles):
    165 
    +    """Dependencies should not show up in status several times when they
    166 
    +    are staged with multiple sysroots"""
    167 
    +
    168 
    +    project = str(datafiles)
    169 
    +
    170 
    +    result = cli.run(project=project,
    171 
    +                     args=['show', '--format', '%{name}', 'manual-integration.bst'])
    172 
    +    result.assert_success()
    173 
    +    pipeline = result.output.splitlines()
    174 
    +    assert pipeline == ['base/base-alpine.bst',
    175 
    +                        'base.bst',
    176 
    +                        'integration.bst',
    177 
    +                        'manual-integration.bst']
    178 
    +
    179 
    +
    180 
    +@pytest.mark.datafiles(DATA_DIR)
    181 
    +def test_sysroot_path_subst_variable(datafiles, cli, tmpdir):
    182 
    +    "Test that variables are expanded in sysroot path"
    183 
    +
    184 
    +    project = str(datafiles)
    185 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    186 
    +
    187 
    +    result = cli.run(project=project,
    188 
    +                     args=['build', 'target-variable.bst'])
    189 
    +    result.assert_success()
    190 
    +
    191 
    +    result = cli.run(project=project,
    192 
    +                     args=['checkout', 'target-variable.bst', checkout])
    193 
    +    result.assert_success()
    194 
    +
    195 
    +    assert os.path.exists(os.path.join(checkout, 'a.txt'))
    196 
    +    assert os.path.exists(os.path.join(checkout, 'path/test', 'b.txt'))

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]