[Notes] [Git][BuildStream/buildstream][jmac/virtual_directory_tests] 9 commits: Fixes to print statements - only use generator once



Title: GitLab

Jim MacArthur pushed to branch jmac/virtual_directory_tests at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/storage/_casbaseddirectory.py
    ... ... @@ -572,8 +572,8 @@ class CasBasedDirectory(Directory):
    572 572
             else:
    
    573 573
                 source_directory = external_pathspec
    
    574 574
                 if files is None:
    
    575
    -                files = list_relative_paths(source_directory)
    
    576
    -            print("Performing import from plain directory {}, containing {} files.".format(source_directory, len(list(files))))
    
    575
    +                files = list(list_relative_paths(source_directory))
    
    576
    +            print("Performing import from plain directory {}, containing {} files: {}".format(source_directory, len(files), files))
    
    577 577
                 result = self._import_files_from_directory(source_directory, files=files)
    
    578 578
     
    
    579 579
             # TODO: No notice is taken of report_written, update_utimes or can_link.
    
    ... ... @@ -591,7 +591,7 @@ class CasBasedDirectory(Directory):
    591 591
             if duplicate_cas:
    
    592 592
                 if duplicate_cas.ref.hash != self.ref.hash:
    
    593 593
                     raise VirtualDirectoryError("Mismatch between file-imported result {} and cas-to-cas imported result {}.".format(duplicate_cas.ref.hash,self.ref.hash))
    
    594
    -
    
    594
    +        print("Import complete; new hash is {}".format(self.ref.hash))
    
    595 595
             return result
    
    596 596
     
    
    597 597
         def set_deterministic_mtime(self):
    
    ... ... @@ -722,6 +722,7 @@ class CasBasedDirectory(Directory):
    722 722
             for (k, v) in self.index.items():
    
    723 723
                 if isinstance(v.buildstream_object, CasBasedDirectory):
    
    724 724
                     filelist.extend([k + os.path.sep + x for x in v.buildstream_object.list_relative_paths()])
    
    725
    +                filelist.append(k)
    
    725 726
                     print("Add directory {}".format(k))
    
    726 727
                 elif isinstance(v.pb_object, remote_execution_pb2.FileNode):
    
    727 728
                     filelist.append(k)
    

  • tests/storage/import_cas.py
    ... ... @@ -17,41 +17,83 @@ class FakeContext():
    17 17
             return []
    
    18 18
     
    
    19 19
     root_filesets = [
    
    20
    -    [('a/b/c/textfile1', 'This is textfile 1\n')],
    
    21
    -    [('a/b/c/textfile1', 'This is the replacement textfile 1\n')],
    
    20
    +    [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
    
    21
    +    [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
    
    22
    +    [('a/b/d', 'D', '')],
    
    23
    +    [('a/b/c', 'S', '/a/b/d')],
    
    24
    +    [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
    
    22 25
     ]
    
    23 26
     
    
    24 27
     empty_hash_ref = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
    
    25 28
     
    
    26 29
     
    
    27 30
     def generate_import_roots(directory):
    
    28
    -    for fileset in [1, 2]:
    
    31
    +    for fileset in range(1, len(root_filesets) + 1):
    
    29 32
             rootname = "root{}".format(fileset)
    
    33
    +        rootdir = os.path.join(directory, "content", rootname)
    
    30 34
     
    
    31
    -        for (path, content) in root_filesets[fileset - 1]:
    
    32
    -            (dirnames, filename) = os.path.split(path)
    
    33
    -            os.makedirs(os.path.join(directory, "content", rootname, dirnames))
    
    35
    +        for (path, typesymbol, content) in root_filesets[fileset - 1]:
    
    36
    +            if typesymbol == 'F':
    
    37
    +                (dirnames, filename) = os.path.split(path)
    
    38
    +                os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
    
    34 39
     
    
    35
    -            with open(os.path.join(directory, "content", rootname, dirnames, filename), "wt") as f:
    
    36
    -                f.write(content)
    
    40
    +                with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
    
    41
    +                    f.write(content)
    
    42
    +            elif typesymbol == 'D':
    
    43
    +                os.makedirs(os.path.join(rootdir, path), exist_ok=True)
    
    44
    +            elif typesymbol == 'S':
    
    45
    +                (dirnames, filename) = os.path.split(path)
    
    46
    +                print("Ensuring the existence of {}".format(os.path.join(rootdir, dirnames)))
    
    47
    +                os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
    
    48
    +                print("attempting to make a symlink called {} pointing to {}".format(path, content))
    
    49
    +                os.symlink(content, os.path.join(rootdir, path))
    
    37 50
     
    
    38 51
     
    
    39
    -def test_cas_import(cli, tmpdir):
    
    52
    +def file_contents(path):
    
    53
    +    with open(path, "r") as f:
    
    54
    +        result = f.read()
    
    55
    +    return result
    
    56
    +
    
    57
    +
    
    58
    +def file_contents_are(path, contents):
    
    59
    +    return file_contents(path) == contents
    
    60
    +
    
    61
    +
    
    62
    +def create_new_vdir(root_number, fake_context, tmpdir):
    
    63
    +    d = CasBasedDirectory(fake_context)
    
    64
    +    d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
    
    65
    +    assert d.ref.hash != empty_hash_ref
    
    66
    +    return d
    
    67
    +
    
    68
    +
    
    69
    +def combinations(integer_range):
    
    70
    +    for x in integer_range:
    
    71
    +        for y in integer_range:
    
    72
    +            yield (x,y)
    
    73
    +
    
    74
    +@pytest.mark.parametrize("roots", combinations([1,2,3,4,5]))
    
    75
    +def test_cas_import(cli, tmpdir, roots):
    
    76
    +    print("Testing import of root {} into root {}".format(roots[0], roots[1]))
    
    40 77
         fake_context = FakeContext()
    
    41 78
         fake_context.artifactdir = tmpdir
    
    42 79
         # Create some fake content
    
    43 80
         generate_import_roots(tmpdir)
    
    44 81
     
    
    45
    -    d = CasBasedDirectory(fake_context)
    
    46
    -    d.import_files(os.path.join(tmpdir, "content", "root1"))
    
    47
    -    assert d.ref.hash != empty_hash_ref
    
    82
    +    (original, overlay) = roots
    
    48 83
     
    
    49
    -    d2 = CasBasedDirectory(fake_context)
    
    50
    -    d2.import_files(os.path.join(tmpdir, "content", "root2"))
    
    51
    -    assert d2.ref.hash != empty_hash_ref
    
    52
    -    print("D2 hash is {}".format(d2.ref.hash))
    
    84
    +    d = create_new_vdir(original, fake_context, tmpdir)
    
    85
    +    d2 = create_new_vdir(overlay, fake_context, tmpdir)
    
    53 86
         d.import_files(d2)
    
    54
    -
    
    55 87
         d.export_files(os.path.join(tmpdir, "output"))
    
    56
    -    assert os.path.exists(os.path.join(tmpdir, "output", "a", "b", "c", "textfile1"))
    
    57
    -    assert file_contents_are(os.path.join(tmpdir, "output", "a", "b", "c", "textfile1"), root_filesets[1][0][1])
    88
    +
    
    89
    +    for item in root_filesets[overlay - 1]:
    
    90
    +        (path, typename, content) = item
    
    91
    +        if typename in ['F', 'S']:
    
    92
    +            assert os.path.lexists(os.path.join(tmpdir, "output", path)), "{} did not exist in the combined virtual directory".format(path)
    
    93
    +        if typename == 'F':
    
    94
    +            assert file_contents_are(os.path.join(tmpdir, "output", path), content)
    
    95
    +        elif typename == 'S':
    
    96
    +            assert os.readlink(os.path.join(tmpdir, "output", path)) == content
    
    97
    +        elif typename == 'D':
    
    98
    +            # Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
    
    99
    +            assert os.path.isdir(os.path.join(tmpdir, "output", path))



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]