[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] WIP: Opening a workspace with a cached build



Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

3 changed files:

Changes:

  • buildstream/_context.py
    ... ... @@ -113,6 +113,9 @@ class Context():
    113 113
             # Whether or not to attempt to pull buildtrees globally
    
    114 114
             self.pullbuildtrees = False
    
    115 115
     
    
    116
    +        # Whether to not include artifact buildtrees in workspaces if available
    
    117
    +        self.workspacebuildtrees = True
    
    118
    +
    
    116 119
             # Private variables
    
    117 120
             self._cache_key = None
    
    118 121
             self._message_handler = None
    
    ... ... @@ -163,7 +166,7 @@ class Context():
    163 166
             _yaml.node_validate(defaults, [
    
    164 167
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    
    165 168
                 'scheduler', 'artifacts', 'logging', 'projects',
    
    166
    -            'cache', 'pullbuildtrees'
    
    169
    +            'cache', 'pullbuildtrees', 'workspacebuildtrees'
    
    167 170
             ])
    
    168 171
     
    
    169 172
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    
    ... ... @@ -191,6 +194,9 @@ class Context():
    191 194
             # Load pull buildtrees configuration
    
    192 195
             self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
    
    193 196
     
    
    197
    +        # Load workspace buildtrees configuration
    
    198
    +        self.workspacebuildtrees = _yaml.node_get(defaults, bool, 'workspacebuildtrees', default_value='True')
    
    199
    +
    
    194 200
             # Load logging config
    
    195 201
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    
    196 202
             _yaml.node_validate(logging, [
    

  • buildstream/_frontend/cli.py
    ... ... @@ -686,12 +686,16 @@ def workspace():
    686 686
                   help="Overwrite files existing in checkout directory")
    
    687 687
     @click.option('--track', 'track_', default=False, is_flag=True,
    
    688 688
                   help="Track and fetch new source references before checking out the workspace")
    
    689
    +@click.option('--no-cache', default=False, is_flag=True,
    
    690
    +              help="Do not checkout the cached buildtree")
    
    689 691
     @click.argument('element',
    
    690 692
                     type=click.Path(readable=False))
    
    691 693
     @click.argument('directory', type=click.Path(file_okay=False))
    
    692 694
     @click.pass_obj
    
    693
    -def workspace_open(app, no_checkout, force, track_, element, directory):
    
    694
    -    """Open a workspace for manual source modification"""
    
    695
    +def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
    
    696
    +    """Open a workspace for manual source modification, the elements buildtree
    
    697
    +    will be provided if available in the local artifact cache.
    
    698
    +    """
    
    695 699
     
    
    696 700
         if os.path.exists(directory):
    
    697 701
     
    
    ... ... @@ -703,11 +707,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
    703 707
                 click.echo("Checkout directory is not empty: {}".format(directory), err=True)
    
    704 708
                 sys.exit(-1)
    
    705 709
     
    
    710
    +    if not no_cache:
    
    711
    +        click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
    
    712
    +
    
    706 713
         with app.initialized():
    
    707 714
             app.stream.workspace_open(element, directory,
    
    708 715
                                       no_checkout=no_checkout,
    
    709 716
                                       track_first=track_,
    
    710
    -                                  force=force)
    
    717
    +                                  force=force,
    
    718
    +                                  no_cache=no_cache)
    
    711 719
     
    
    712 720
     
    
    713 721
     ##################################################################
    

  • buildstream/_stream.py
    ... ... @@ -456,11 +456,17 @@ class Stream():
    456 456
         #    no_checkout (bool): Whether to skip checking out the source
    
    457 457
         #    track_first (bool): Whether to track and fetch first
    
    458 458
         #    force (bool): Whether to ignore contents in an existing directory
    
    459
    +    #    no_cache (bool): Whether to not include the cached buildtree
    
    459 460
         #
    
    460 461
         def workspace_open(self, target, directory, *,
    
    461 462
                            no_checkout,
    
    462 463
                            track_first,
    
    463
    -                       force):
    
    464
    +                       force,
    
    465
    +                       no_cache):
    
    466
    +
    
    467
    +        # Override no_cache if the global user conf workspacebuildtrees is false
    
    468
    +        if not self._context.workspacebuildtrees:
    
    469
    +            no_cache = True
    
    464 470
     
    
    465 471
             if track_first:
    
    466 472
                 track_targets = (target,)
    
    ... ... @@ -473,7 +479,21 @@ class Stream():
    473 479
             target = elements[0]
    
    474 480
             directory = os.path.abspath(directory)
    
    475 481
     
    
    476
    -        if not list(target.sources()):
    
    482
    +        # Check if given target has a buildtree artifact cached locally
    
    483
    +        buildtree = None
    
    484
    +        if target._cached():
    
    485
    +            buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key, 'buildtree')
    
    486
    +
    
    487
    +        # If we're running in the default state, make the user aware of buildtree usage
    
    488
    +        if not no_cache:
    
    489
    +            if buildtree:
    
    490
    +                self._message(MessageType.INFO, "{}'s buildtree artifact is available"
    
    491
    +                              "workspace will be opened with it".format(target.normal_name))
    
    492
    +            else:
    
    493
    +                self._message(MessageType.WARN, "{}'s buildtree artifact not available"
    
    494
    +                              "workspace will be opened with source checkout".format(target.normal_name))
    
    495
    +
    
    496
    +        if (not buildtree or no_cache) and not list(target.sources()):
    
    477 497
                 build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    
    478 498
                 if not build_depends:
    
    479 499
                     raise StreamError("The given element has no sources")
    
    ... ... @@ -492,18 +512,21 @@ class Stream():
    492 512
             # If we're going to checkout, we need at least a fetch,
    
    493 513
             # if we were asked to track first, we're going to fetch anyway.
    
    494 514
             #
    
    495
    -        if not no_checkout or track_first:
    
    496
    -            track_elements = []
    
    497
    -            if track_first:
    
    498
    -                track_elements = elements
    
    499
    -            self._fetch(elements, track_elements=track_elements)
    
    500
    -
    
    501
    -        if not no_checkout and target._get_consistency() != Consistency.CACHED:
    
    502
    -            raise StreamError("Could not stage uncached source. " +
    
    503
    -                              "Use `--track` to track and " +
    
    504
    -                              "fetch the latest version of the " +
    
    505
    -                              "source.")
    
    506
    -
    
    515
    +        if not buildtree or no_cache:
    
    516
    +            if not no_checkout or track_first:
    
    517
    +                track_elements = []
    
    518
    +                if track_first:
    
    519
    +                    track_elements = elements
    
    520
    +                self._fetch(elements, track_elements=track_elements)
    
    521
    +
    
    522
    +        if not buildtree or no_cache:
    
    523
    +            if not no_checkout and target._get_consistency() != Consistency.CACHED:
    
    524
    +                raise StreamError("Could not stage uncached source. " +
    
    525
    +                                  "Use `--track` to track and " +
    
    526
    +                                  "fetch the latest version of the " +
    
    527
    +                                  "source.")
    
    528
    +
    
    529
    +        # Presume workspace to be forced if previous StreamError not raised
    
    507 530
             if workspace:
    
    508 531
                 workspaces.delete_workspace(target._get_full_name())
    
    509 532
                 workspaces.save_config()
    
    ... ... @@ -515,7 +538,7 @@ class Stream():
    515 538
     
    
    516 539
             workspaces.create_workspace(target._get_full_name(), directory)
    
    517 540
     
    
    518
    -        if not no_checkout:
    
    541
    +        if (not buildtree or no_cache) and not no_checkout:
    
    519 542
                 with target.timed_activity("Staging sources to {}".format(directory)):
    
    520 543
                     target._open_workspace()
    
    521 544
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]