GNOME.org
 

[Notes] [Git][BuildStream/buildstream][chandan/sourcetransform] fixup! Allow source plugins to access previous sources

Title: GitLab

Chandan Singh pushed to branch chandan/sourcetransform at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/source.py
    ... ... @@ -125,7 +125,26 @@ class Source(Plugin):
    125 125 
         __defaults_set = False   # Flag, in case there are not defaults at all
    126 126
    127 127 
         requires_previous_sources_track = False
    128 
    +    """Whether access to previous sources is required during track
    129 
    +
    130 
    +    When set to True:
    131 
    +      o all sources listed before this source in the given element will be
    132 
    +        fetched before this source is tracked
    133 
    +      o Source.track() will be called with an additional keywork argument
    134 
    +        `previous_sources` that will contain a list of sources
    135 
    +      o this source can not be the first source for an element
    136 
    +    """
    137 
    +
    128 138 
         requires_previous_sources_fetch = False
    139 
    +    """Whether access to previous sources is required during fetch
    140 
    +
    141 
    +    When set to True:
    142 
    +      o all sources listed before this source in the given element will be
    143 
    +        fetched before this source is fetched
    144 
    +      o Source.fetch() will be called with an additional keyword argument
    145 
    +        `previous_sources` that will contain a list of sources
    146 
    +      o this source can not be the first source for an element
    147 
    +    """
    129 148
    130 149 
         def __init__(self, context, project, meta):
    131 150 
             provenance = _yaml.node_get_provenance(meta.config)

  • tests/sources/previous_source_access/plugins/sources/foo_transform.py
    1 
    +"""
    2 
    +foo_transform - transform "file" from previous sources into "filetransform"
    3 
    +===========================================================================
    4 
    +
    5 
    +This is a test source plugin that looks for a file named "file" staged by
    6 
    +previous sources, and copies its contents to a file called "filetransform".
    7 
    +
    8 
    +"""
    9 
    +
    1 10 
     import os
    2 11 
     import hashlib
    3 12
    ... ... @@ -5,23 +14,25 @@ from buildstream import Consistency, Source, SourceError, utils
    5 14
    6 15
    7 16 
     class FooTransformSource(Source):
    8 
    -    """Test plugin that copies "file" from previous source as "filetransform".
    9 17
    10 
    -    """
    11 
    -    requires_previous_sources_fetch = True
    18 
    +    # We need access to previous both at track time and fetch time
    12 19 
         requires_previous_sources_track = True
    13 
    -
    14 
    -    def configure(self, node):
    15 
    -        self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
    16 
    -        self.ref = self.node_get_member(node, str, 'ref', None)
    20 
    +    requires_previous_sources_fetch = True
    17 21
    18 22 
         @property
    19 23 
         def mirror(self):
    24 
    +        """Directory where this source should stage its files
    25 
    +
    26 
    +        """
    20 27 
             path = os.path.join(self.get_mirror_directory(), self.name,
    21 28 
                                 self.ref.strip())
    22 29 
             os.makedirs(path, exist_ok=True)
    23 30 
             return path
    24 31
    32 
    +    def configure(self, node):
    33 
    +        self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
    34 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    35 
    +
    25 36 
         def preflight(self):
    26 37 
             pass
    27 38
    ... ... @@ -31,6 +42,8 @@ class FooTransformSource(Source):
    31 42 
         def get_consistency(self):
    32 43 
             if self.ref is None:
    33 44 
                 return Consistency.INCONSISTENT
    45 
    +        # If we have a file called "filetransform", verify that its checksum
    46 
    +        # matches our ref. Otherwise, it resolved but not cached.
    34 47 
             fpath = os.path.join(self.mirror, 'filetransform')
    35 48 
             try:
    36 49 
                 with open(fpath, 'rb') as f:
    ... ... @@ -47,19 +60,25 @@ class FooTransformSource(Source):
    47 60 
             self.ref = node['ref'] = ref
    48 61
    49 62 
         def track(self, previous_sources_dir):
    63 
    +        # Store the checksum of the file from previous source as our ref
    50 64 
             fpath = os.path.join(previous_sources_dir, 'file')
    51 65 
             with open(fpath, 'rb') as f:
    52 66 
                 return hashlib.sha256(f.read()).hexdigest()
    53 67
    54 68 
         def fetch(self, previous_sources_dir):
    55 69 
             fpath = os.path.join(previous_sources_dir, 'file')
    70 
    +        # Verify that the checksum of the file from previous source matches
    71 
    +        # our ref
    56 72 
             with open(fpath, 'rb') as f:
    57 73 
                 if hashlib.sha256(f.read()).hexdigest() != self.ref.strip():
    58 74 
                     raise SourceError("Element references do not match")
    75 
    +
    76 
    +        # Copy "file" as "filetransform"
    59 77 
             newfpath = os.path.join(self.mirror, 'filetransform')
    60 78 
             utils.safe_copy(fpath, newfpath)
    61 79
    62 80 
         def stage(self, directory):
    81 
    +        # Simply stage the "filetransform" file
    63 82 
             utils.safe_copy(os.path.join(self.mirror, 'filetransform'),
    64 83 
                             os.path.join(directory, 'filetransform'))
    65 84

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]