Jim MacArthur pushed to branch jmac/vdir_import_test at BuildStream / buildstream
Commits:
-
7e984dff
by Jim MacArthur at 2018-10-04T16:52:15Z
-
c15e3931
by Jim MacArthur at 2018-10-05T12:40:57Z
-
99f6e97c
by Jim MacArthur at 2018-10-05T12:43:49Z
-
7cf28a8d
by Jim MacArthur at 2018-10-05T12:59:34Z
2 changed files:
Changes:
| ... | ... | @@ -280,7 +280,9 @@ class CasBasedDirectory(Directory): |
| 280 | 280 |
directory = root
|
| 281 | 281 |
components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
|
| 282 | 282 |
for c in components:
|
| 283 |
- if c == "..":
|
|
| 283 |
+ if c == ".":
|
|
| 284 |
+ pass
|
|
| 285 |
+ elif c == "..":
|
|
| 284 | 286 |
directory = directory.parent
|
| 285 | 287 |
else:
|
| 286 | 288 |
directory = directory.descend(c, create=True)
|
| ... | ... | @@ -321,7 +323,9 @@ class CasBasedDirectory(Directory): |
| 321 | 323 |
# We ran out of path elements and ended up in a directory
|
| 322 | 324 |
return directory
|
| 323 | 325 |
c = components.pop(0)
|
| 324 |
- if c == "..":
|
|
| 326 |
+ if c == ".":
|
|
| 327 |
+ pass
|
|
| 328 |
+ elif c == "..":
|
|
| 325 | 329 |
print(" resolving {}: up-dir".format(c))
|
| 326 | 330 |
# If directory.parent *is* None, this is an attempt to access
|
| 327 | 331 |
# '..' from the root, which is valid under POSIX; it just
|
| ... | ... | @@ -610,16 +614,18 @@ class CasBasedDirectory(Directory): |
| 610 | 614 |
symlink_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.SymlinkNode), self.index.items()))
|
| 611 | 615 |
file_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.FileNode), self.index.items()))
|
| 612 | 616 |
directory_list = list(filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory), self.index.items()))
|
| 617 |
+ symlinks_to_directories_list = []
|
|
| 613 | 618 |
print("Running list_relative_paths on relpath {}. files={}, symlinks={}".format(relpath, [f[0] for f in file_list], [s[0] for s in symlink_list]))
|
| 614 | 619 |
|
| 615 | 620 |
for (k, v) in sorted(symlink_list):
|
| 616 | 621 |
target = self._resolve(k, absolute_symlinks_resolve=True)
|
| 617 | 622 |
if isinstance(target, CasBasedDirectory):
|
| 618 |
- print("Adding the resolved symlink {} which resolves to {} to our directory list".format(k, target))
|
|
| 619 |
- directory_list.append((k,IndexEntry(k, buildstream_object=target)))
|
|
| 623 |
+ symlinks_to_directories_list.append(k)
|
|
| 620 | 624 |
else:
|
| 621 | 625 |
# Broken symlinks are also considered files!
|
| 622 | 626 |
file_list.append((k,v))
|
| 627 |
+ for d in sorted(symlinks_to_directories_list):
|
|
| 628 |
+ yield os.path.join(relpath, d)
|
|
| 623 | 629 |
if file_list == [] and relpath != "":
|
| 624 | 630 |
print("Yielding empty directory name {}".format(relpath))
|
| 625 | 631 |
yield relpath
|
| 1 | 1 |
import os
|
| 2 | 2 |
import pytest
|
| 3 |
+import random
|
|
| 3 | 4 |
from tests.testutils import cli
|
| 4 | 5 |
|
| 5 | 6 |
from buildstream.storage import CasBasedDirectory
|
| ... | ... | @@ -27,6 +28,7 @@ root_filesets = [ |
| 27 | 28 |
]
|
| 28 | 29 |
|
| 29 | 30 |
empty_hash_ref = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 31 |
+RANDOM_SEED = 69105
|
|
| 30 | 32 |
|
| 31 | 33 |
|
| 32 | 34 |
def generate_import_roots(directory):
|
| ... | ... | @@ -48,6 +50,33 @@ def generate_import_roots(directory): |
| 48 | 50 |
os.symlink(content, os.path.join(rootdir, path))
|
| 49 | 51 |
|
| 50 | 52 |
|
| 53 |
+def generate_random_root(directory):
|
|
| 54 |
+ random.seed(RANDOM_SEED)
|
|
| 55 |
+ rootname = "root6"
|
|
| 56 |
+ rootdir = os.path.join(directory, "content", rootname)
|
|
| 57 |
+ things = []
|
|
| 58 |
+ locations = ['.']
|
|
| 59 |
+ for i in range(0, 100):
|
|
| 60 |
+ location = random.choice(locations)
|
|
| 61 |
+ thingname = "node{}".format(i)
|
|
| 62 |
+ thing = random.choice(['dir', 'link', 'file'])
|
|
| 63 |
+ target = os.path.join(rootdir, location, thingname)
|
|
| 64 |
+ if thing == 'dir':
|
|
| 65 |
+ os.makedirs(target)
|
|
| 66 |
+ locations.append(os.path.join(location, thingname))
|
|
| 67 |
+ elif thing == 'file':
|
|
| 68 |
+ with open(target, "wt") as f:
|
|
| 69 |
+ f.write("This is node {}\n".format(i))
|
|
| 70 |
+ elif thing == 'link':
|
|
| 71 |
+ # TODO: Make some relative symlinks
|
|
| 72 |
+ if random.randint(1, 3) == 1 or len(things) == 0:
|
|
| 73 |
+ os.symlink("/broken", target)
|
|
| 74 |
+ else:
|
|
| 75 |
+ os.symlink(random.choice(things), target)
|
|
| 76 |
+ things.append(os.path.join(location, thingname))
|
|
| 77 |
+ print("Generated {}/{} ".format(rootdir, things[-1]))
|
|
| 78 |
+ |
|
| 79 |
+ |
|
| 51 | 80 |
def file_contents(path):
|
| 52 | 81 |
with open(path, "r") as f:
|
| 53 | 82 |
result = f.read()
|
| ... | ... | @@ -64,6 +93,7 @@ def create_new_casdir(root_number, fake_context, tmpdir): |
| 64 | 93 |
assert d.ref.hash != empty_hash_ref
|
| 65 | 94 |
return d
|
| 66 | 95 |
|
| 96 |
+ |
|
| 67 | 97 |
def create_new_filedir(root_number, tmpdir):
|
| 68 | 98 |
root = os.path.join(tmpdir, "vdir")
|
| 69 | 99 |
os.makedirs(root)
|
| ... | ... | @@ -117,7 +147,7 @@ def test_cas_import(cli, tmpdir, original, overlay): |
| 117 | 147 |
fake_context.artifactdir = tmpdir
|
| 118 | 148 |
# Create some fake content
|
| 119 | 149 |
generate_import_roots(tmpdir)
|
| 120 |
- |
|
| 150 |
+ generate_random_root(tmpdir)
|
|
| 121 | 151 |
d = create_new_casdir(original, fake_context, tmpdir)
|
| 122 | 152 |
d2 = create_new_casdir(overlay, fake_context, tmpdir)
|
| 123 | 153 |
d.import_files(d2)
|
| ... | ... | @@ -144,13 +174,20 @@ def test_cas_import(cli, tmpdir, original, overlay): |
| 144 | 174 |
# Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
|
| 145 | 175 |
assert os.path.isdir(realpath)
|
| 146 | 176 |
|
| 177 |
+ # Now do the same thing with filebaseddirectories and check the contents match
|
|
| 178 |
+ d3 = create_new_casdir(original, fake_context, tmpdir)
|
|
| 179 |
+ d4 = create_new_filedir(overlay, tmpdir)
|
|
| 180 |
+ d3.import_files(d2)
|
|
| 181 |
+ assert d.ref.hash == d3.ref.hash
|
|
| 182 |
+ |
|
| 147 | 183 |
|
| 148 |
-@pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
|
|
| 184 |
+@pytest.mark.parametrize("root", [1, 2, 3, 4, 5, 6])
|
|
| 149 | 185 |
def test_directory_listing(cli, tmpdir, root):
|
| 150 | 186 |
fake_context = FakeContext()
|
| 151 | 187 |
fake_context.artifactdir = tmpdir
|
| 152 | 188 |
# Create some fake content
|
| 153 | 189 |
generate_import_roots(tmpdir)
|
| 190 |
+ generate_random_root(tmpdir)
|
|
| 154 | 191 |
|
| 155 | 192 |
d = create_new_filedir(root, tmpdir)
|
| 156 | 193 |
filelist = list(d.list_relative_paths())
|
| ... | ... | @@ -162,4 +199,4 @@ def test_directory_listing(cli, tmpdir, root): |
| 162 | 199 |
print("{}".format(filelist))
|
| 163 | 200 |
print("filelist for root {} via CasBasedDirectory:".format(root))
|
| 164 | 201 |
print("{}".format(filelist2))
|
| 165 |
- assert(filelist==filelist2)
|
|
| 202 |
+ assert filelist == filelist2
|