Tristan Van Berkom pushed to branch master at BuildStream / buildstream
Commits:
-
b1d2f001
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
ee7fc47f
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
5fc9a1da
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
50cb2706
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
f139ca8f
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
afe1aa9f
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
57300201
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
a0297625
by Tristan Van Berkom at 2018-12-06T14:18:31Z
-
60e96781
by Tristan Van Berkom at 2018-12-06T15:14:37Z
5 changed files:
- buildstream/_versions.py
- buildstream/plugins/sources/git.py
- buildstream/source.py
- tests/sources/git.py
- tests/testutils/repo/git.py
Changes:
... | ... | @@ -23,7 +23,7 @@ |
23 | 23 |
# This version is bumped whenever enhancements are made
|
24 | 24 |
# to the `project.conf` format or the core element format.
|
25 | 25 |
#
|
26 |
-BST_FORMAT_VERSION = 19
|
|
26 |
+BST_FORMAT_VERSION = 20
|
|
27 | 27 |
|
28 | 28 |
|
29 | 29 |
# The base BuildStream artifact version
|
... | ... | @@ -133,7 +133,22 @@ details on common configuration options for sources. |
133 | 133 |
|
134 | 134 |
This plugin provides the following :ref:`configurable warnings <configurable_warnings>`:
|
135 | 135 |
|
136 |
-- ``git:inconsistent-submodule`` - A submodule was found to be missing from the underlying git repository.
|
|
136 |
+- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never
|
|
137 |
+ added with `git submodule add`.
|
|
138 |
+ |
|
139 |
+- ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in
|
|
140 |
+ the source configuration and was not disabled for checkout.
|
|
141 |
+ |
|
142 |
+ .. note::
|
|
143 |
+ |
|
144 |
+ The ``git:unlisted-submodule`` warning is available since :ref:`format version 20 <project_format_version>`
|
|
145 |
+ |
|
146 |
+- ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist
|
|
147 |
+ in the repository.
|
|
148 |
+ |
|
149 |
+ .. note::
|
|
150 |
+ |
|
151 |
+ The ``git:invalid-submodule`` warning is available since :ref:`format version 20 <project_format_version>`
|
|
137 | 152 |
|
138 | 153 |
This plugin also utilises the following configurable :class:`core warnings <buildstream.types.CoreWarnings>`:
|
139 | 154 |
|
... | ... | @@ -158,6 +173,8 @@ GIT_MODULES = '.gitmodules' |
158 | 173 |
|
159 | 174 |
# Warnings
|
160 | 175 |
WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule"
|
176 |
+WARN_UNLISTED_SUBMODULE = "unlisted-submodule"
|
|
177 |
+WARN_INVALID_SUBMODULE = "invalid-submodule"
|
|
161 | 178 |
|
162 | 179 |
|
163 | 180 |
# Because of handling of submodules, we maintain a GitMirror
|
... | ... | @@ -305,7 +322,7 @@ class GitMirror(SourceFetcher): |
305 | 322 |
|
306 | 323 |
return ref, list(tags)
|
307 | 324 |
|
308 |
- def stage(self, directory, track=None):
|
|
325 |
+ def stage(self, directory):
|
|
309 | 326 |
fullpath = os.path.join(directory, self.path)
|
310 | 327 |
|
311 | 328 |
# Using --shared here avoids copying the objects into the checkout, in any
|
... | ... | @@ -324,11 +341,7 @@ class GitMirror(SourceFetcher): |
324 | 341 |
|
325 | 342 |
self._rebuild_git(fullpath)
|
326 | 343 |
|
327 |
- # Check that the user specified ref exists in the track if provided & not already tracked
|
|
328 |
- if track:
|
|
329 |
- self.assert_ref_in_track(fullpath, track)
|
|
330 |
- |
|
331 |
- def init_workspace(self, directory, track=None):
|
|
344 |
+ def init_workspace(self, directory):
|
|
332 | 345 |
fullpath = os.path.join(directory, self.path)
|
333 | 346 |
url = self.source.translate_url(self.url)
|
334 | 347 |
|
... | ... | @@ -344,10 +357,6 @@ class GitMirror(SourceFetcher): |
344 | 357 |
fail="Failed to checkout git ref {}".format(self.ref),
|
345 | 358 |
cwd=fullpath)
|
346 | 359 |
|
347 |
- # Check that the user specified ref exists in the track if provided & not already tracked
|
|
348 |
- if track:
|
|
349 |
- self.assert_ref_in_track(fullpath, track)
|
|
350 |
- |
|
351 | 360 |
# List the submodules (path/url tuples) present at the given ref of this repo
|
352 | 361 |
def submodule_list(self):
|
353 | 362 |
modules = "{}:{}".format(self.ref, GIT_MODULES)
|
... | ... | @@ -413,28 +422,6 @@ class GitMirror(SourceFetcher): |
413 | 422 |
|
414 | 423 |
return None
|
415 | 424 |
|
416 |
- # Assert that ref exists in track, if track has been specified.
|
|
417 |
- def assert_ref_in_track(self, fullpath, track):
|
|
418 |
- _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track,
|
|
419 |
- '--contains', self.ref],
|
|
420 |
- cwd=fullpath,)
|
|
421 |
- if branch:
|
|
422 |
- return
|
|
423 |
- else:
|
|
424 |
- _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track,
|
|
425 |
- '--contains', self.ref],
|
|
426 |
- cwd=fullpath,)
|
|
427 |
- if tag:
|
|
428 |
- return
|
|
429 |
- |
|
430 |
- detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \
|
|
431 |
- "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \
|
|
432 |
- "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url)
|
|
433 |
- |
|
434 |
- self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
|
|
435 |
- .format(self.source, self.ref, track, self.url),
|
|
436 |
- detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
|
|
437 |
- |
|
438 | 425 |
def _rebuild_git(self, fullpath):
|
439 | 426 |
if not self.tags:
|
440 | 427 |
return
|
... | ... | @@ -563,7 +550,6 @@ class GitSource(Source): |
563 | 550 |
self.submodule_checkout_overrides[path] = checkout
|
564 | 551 |
|
565 | 552 |
self.mark_download_url(self.original_url)
|
566 |
- self.tracked = False
|
|
567 | 553 |
|
568 | 554 |
def preflight(self):
|
569 | 555 |
# Check if git is installed, get the binary at the same time
|
... | ... | @@ -653,8 +639,6 @@ class GitSource(Source): |
653 | 639 |
# Update self.mirror.ref and node.ref from the self.tracking branch
|
654 | 640 |
ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
|
655 | 641 |
|
656 |
- # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
|
|
657 |
- self.tracked = True
|
|
658 | 642 |
return ret
|
659 | 643 |
|
660 | 644 |
def init_workspace(self, directory):
|
... | ... | @@ -662,7 +646,7 @@ class GitSource(Source): |
662 | 646 |
self.refresh_submodules()
|
663 | 647 |
|
664 | 648 |
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
|
665 |
- self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None))
|
|
649 |
+ self.mirror.init_workspace(directory)
|
|
666 | 650 |
for mirror in self.submodules:
|
667 | 651 |
mirror.init_workspace(directory)
|
668 | 652 |
|
... | ... | @@ -678,15 +662,9 @@ class GitSource(Source): |
678 | 662 |
# Stage the main repo in the specified directory
|
679 | 663 |
#
|
680 | 664 |
with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
|
681 |
- self.mirror.stage(directory, track=(self.tracking if not self.tracked else None))
|
|
665 |
+ self.mirror.stage(directory)
|
|
682 | 666 |
for mirror in self.submodules:
|
683 |
- if mirror.path in self.submodule_checkout_overrides:
|
|
684 |
- checkout = self.submodule_checkout_overrides[mirror.path]
|
|
685 |
- else:
|
|
686 |
- checkout = self.checkout_submodules
|
|
687 |
- |
|
688 |
- if checkout:
|
|
689 |
- mirror.stage(directory)
|
|
667 |
+ mirror.stage(directory)
|
|
690 | 668 |
|
691 | 669 |
def get_source_fetchers(self):
|
692 | 670 |
yield self.mirror
|
... | ... | @@ -694,6 +672,74 @@ class GitSource(Source): |
694 | 672 |
for submodule in self.submodules:
|
695 | 673 |
yield submodule
|
696 | 674 |
|
675 |
+ def validate_cache(self):
|
|
676 |
+ discovered_submodules = {}
|
|
677 |
+ unlisted_submodules = []
|
|
678 |
+ invalid_submodules = []
|
|
679 |
+ |
|
680 |
+ for path, url in self.mirror.submodule_list():
|
|
681 |
+ discovered_submodules[path] = url
|
|
682 |
+ if self.ignore_submodule(path):
|
|
683 |
+ continue
|
|
684 |
+ |
|
685 |
+ override_url = self.submodule_overrides.get(path)
|
|
686 |
+ if not override_url:
|
|
687 |
+ unlisted_submodules.append((path, url))
|
|
688 |
+ |
|
689 |
+ # Warn about submodules which are explicitly configured but do not exist
|
|
690 |
+ for path, url in self.submodule_overrides.items():
|
|
691 |
+ if path not in discovered_submodules:
|
|
692 |
+ invalid_submodules.append((path, url))
|
|
693 |
+ |
|
694 |
+ if invalid_submodules:
|
|
695 |
+ detail = []
|
|
696 |
+ for path, url in invalid_submodules:
|
|
697 |
+ detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
|
|
698 |
+ |
|
699 |
+ self.warn("{}: Invalid submodules specified".format(self),
|
|
700 |
+ warning_token=WARN_INVALID_SUBMODULE,
|
|
701 |
+ detail="The following submodules are specified in the source "
|
|
702 |
+ "description but do not exist according to the repository\n\n" +
|
|
703 |
+ "\n".join(detail))
|
|
704 |
+ |
|
705 |
+ # Warn about submodules which exist but have not been explicitly configured
|
|
706 |
+ if unlisted_submodules:
|
|
707 |
+ detail = []
|
|
708 |
+ for path, url in unlisted_submodules:
|
|
709 |
+ detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
|
|
710 |
+ |
|
711 |
+ self.warn("{}: Unlisted submodules exist".format(self),
|
|
712 |
+ warning_token=WARN_UNLISTED_SUBMODULE,
|
|
713 |
+ detail="The following submodules exist but are not specified " +
|
|
714 |
+ "in the source description\n\n" +
|
|
715 |
+ "\n".join(detail))
|
|
716 |
+ |
|
717 |
+ # Assert that the ref exists in the track tag/branch, if track has been specified.
|
|
718 |
+ ref_in_track = False
|
|
719 |
+ if self.tracking:
|
|
720 |
+ _, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking,
|
|
721 |
+ '--contains', self.mirror.ref],
|
|
722 |
+ cwd=self.mirror.mirror)
|
|
723 |
+ if branch:
|
|
724 |
+ ref_in_track = True
|
|
725 |
+ else:
|
|
726 |
+ _, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking,
|
|
727 |
+ '--contains', self.mirror.ref],
|
|
728 |
+ cwd=self.mirror.mirror)
|
|
729 |
+ if tag:
|
|
730 |
+ ref_in_track = True
|
|
731 |
+ |
|
732 |
+ if not ref_in_track:
|
|
733 |
+ detail = "The ref provided for the element does not exist locally " + \
|
|
734 |
+ "in the provided track branch / tag '{}'.\n".format(self.tracking) + \
|
|
735 |
+ "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + \
|
|
736 |
+ "with `bst track`,\n" + \
|
|
737 |
+ "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
|
|
738 |
+ |
|
739 |
+ self.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
|
|
740 |
+ .format(self, self.mirror.ref, self.tracking, self.mirror.url),
|
|
741 |
+ detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
|
|
742 |
+ |
|
697 | 743 |
###########################################################
|
698 | 744 |
# Local Functions #
|
699 | 745 |
###########################################################
|
... | ... | @@ -718,12 +764,12 @@ class GitSource(Source): |
718 | 764 |
self.mirror.ensure()
|
719 | 765 |
submodules = []
|
720 | 766 |
|
721 |
- # XXX Here we should issue a warning if either:
|
|
722 |
- # A.) A submodule exists but is not defined in the element configuration
|
|
723 |
- # B.) The element configuration configures submodules which dont exist at the current ref
|
|
724 |
- #
|
|
725 | 767 |
for path, url in self.mirror.submodule_list():
|
726 | 768 |
|
769 |
+ # Completely ignore submodules which are disabled for checkout
|
|
770 |
+ if self.ignore_submodule(path):
|
|
771 |
+ continue
|
|
772 |
+ |
|
727 | 773 |
# Allow configuration to override the upstream
|
728 | 774 |
# location of the submodules.
|
729 | 775 |
override_url = self.submodule_overrides.get(path)
|
... | ... | @@ -747,6 +793,16 @@ class GitSource(Source): |
747 | 793 |
tags.append((tag, commit_ref, annotated))
|
748 | 794 |
return tags
|
749 | 795 |
|
796 |
+ # Checks whether the plugin configuration has explicitly
|
|
797 |
+ # configured this submodule to be ignored
|
|
798 |
+ def ignore_submodule(self, path):
|
|
799 |
+ try:
|
|
800 |
+ checkout = self.submodule_checkout_overrides[path]
|
|
801 |
+ except KeyError:
|
|
802 |
+ checkout = self.checkout_submodules
|
|
803 |
+ |
|
804 |
+ return not checkout
|
|
805 |
+ |
|
750 | 806 |
|
751 | 807 |
# Plugin entry point
|
752 | 808 |
def setup():
|
... | ... | @@ -102,6 +102,11 @@ these methods are mandatory to implement. |
102 | 102 |
submodules). For details on how to define a SourceFetcher, see
|
103 | 103 |
:ref:`SourceFetcher <core_source_fetcher>`.
|
104 | 104 |
|
105 |
+* :func:`Source.validate_cache() <buildstream.source.Source.validate_cache>`
|
|
106 |
+ |
|
107 |
+ Perform any validations which require the sources to be cached.
|
|
108 |
+ |
|
109 |
+ **Optional**: This is completely optional and will do nothing if left unimplemented.
|
|
105 | 110 |
|
106 | 111 |
Accessing previous sources
|
107 | 112 |
--------------------------
|
... | ... | @@ -480,9 +485,22 @@ class Source(Plugin): |
480 | 485 |
|
481 | 486 |
*Since: 1.2*
|
482 | 487 |
"""
|
483 |
- |
|
484 | 488 |
return []
|
485 | 489 |
|
490 |
+ def validate_cache(self):
|
|
491 |
+ """Implement any validations once we know the sources are cached
|
|
492 |
+ |
|
493 |
+ This is guaranteed to be called only once for a given session
|
|
494 |
+ once the sources are known to be
|
|
495 |
+ :attr:`Consistency.CACHED <buildstream.types.Consistency.CACHED>`,
|
|
496 |
+ if source tracking is enabled in the session for this source,
|
|
497 |
+ then this will only be called if the sources become cached after
|
|
498 |
+ tracking completes.
|
|
499 |
+ |
|
500 |
+ *Since: 1.4*
|
|
501 |
+ """
|
|
502 |
+ pass
|
|
503 |
+ |
|
486 | 504 |
#############################################################
|
487 | 505 |
# Public Methods #
|
488 | 506 |
#############################################################
|
... | ... | @@ -659,6 +677,11 @@ class Source(Plugin): |
659 | 677 |
with context.silence():
|
660 | 678 |
self.__consistency = self.get_consistency() # pylint: disable=assignment-from-no-return
|
661 | 679 |
|
680 |
+ # Give the Source an opportunity to validate the cached
|
|
681 |
+ # sources as soon as the Source becomes Consistency.CACHED.
|
|
682 |
+ if self.__consistency == Consistency.CACHED:
|
|
683 |
+ self.validate_cache()
|
|
684 |
+ |
|
662 | 685 |
# Return cached consistency
|
663 | 686 |
#
|
664 | 687 |
def _get_consistency(self):
|
... | ... | @@ -455,6 +455,274 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail): |
455 | 455 |
assert "ref-not-in-track" in result.stderr
|
456 | 456 |
|
457 | 457 |
|
458 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
459 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
460 |
+@pytest.mark.parametrize("fail", ['warn', 'error'])
|
|
461 |
+def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
|
|
462 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
463 |
+ |
|
464 |
+ # Make the warning an error if we're testing errors
|
|
465 |
+ if fail == 'error':
|
|
466 |
+ project_template = {
|
|
467 |
+ "name": "foo",
|
|
468 |
+ "fatal-warnings": ['git:unlisted-submodule']
|
|
469 |
+ }
|
|
470 |
+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
|
|
471 |
+ |
|
472 |
+ # Create the submodule first from the 'subrepofiles' subdir
|
|
473 |
+ subrepo = create_repo('git', str(tmpdir), 'subrepo')
|
|
474 |
+ subrepo.create(os.path.join(project, 'subrepofiles'))
|
|
475 |
+ |
|
476 |
+ # Create the repo from 'repofiles' subdir
|
|
477 |
+ repo = create_repo('git', str(tmpdir))
|
|
478 |
+ ref = repo.create(os.path.join(project, 'repofiles'))
|
|
479 |
+ |
|
480 |
+ # Add a submodule pointing to the one we created
|
|
481 |
+ ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
|
|
482 |
+ |
|
483 |
+ # Create the source, and delete the explicit configuration
|
|
484 |
+ # of the submodules.
|
|
485 |
+ #
|
|
486 |
+ # We expect this to cause an unlisted submodule warning
|
|
487 |
+ # after the source has been fetched.
|
|
488 |
+ #
|
|
489 |
+ gitsource = repo.source_config(ref=ref)
|
|
490 |
+ del gitsource['submodules']
|
|
491 |
+ |
|
492 |
+ # Write out our test target
|
|
493 |
+ element = {
|
|
494 |
+ 'kind': 'import',
|
|
495 |
+ 'sources': [
|
|
496 |
+ gitsource
|
|
497 |
+ ]
|
|
498 |
+ }
|
|
499 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
500 |
+ |
|
501 |
+ # We will not see the warning or error before the first fetch, because
|
|
502 |
+ # we don't have the repository yet and so we have no knowledge of
|
|
503 |
+ # the unlisted submodule.
|
|
504 |
+ result = cli.run(project=project, args=['show', 'target.bst'])
|
|
505 |
+ result.assert_success()
|
|
506 |
+ assert "git:unlisted-submodule" not in result.stderr
|
|
507 |
+ |
|
508 |
+ # We will notice this directly in fetch, as it will try to fetch
|
|
509 |
+ # the submodules it discovers as a result of fetching the primary repo.
|
|
510 |
+ result = cli.run(project=project, args=['fetch', 'target.bst'])
|
|
511 |
+ |
|
512 |
+ # Assert a warning or an error depending on what we're checking
|
|
513 |
+ if fail == 'error':
|
|
514 |
+ result.assert_main_error(ErrorDomain.STREAM, None)
|
|
515 |
+ result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
|
|
516 |
+ else:
|
|
517 |
+ result.assert_success()
|
|
518 |
+ assert "git:unlisted-submodule" in result.stderr
|
|
519 |
+ |
|
520 |
+ # Now that we've fetched it, `bst show` will discover the unlisted submodule too
|
|
521 |
+ result = cli.run(project=project, args=['show', 'target.bst'])
|
|
522 |
+ |
|
523 |
+ # Assert a warning or an error depending on what we're checking
|
|
524 |
+ if fail == 'error':
|
|
525 |
+ result.assert_main_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
|
|
526 |
+ else:
|
|
527 |
+ result.assert_success()
|
|
528 |
+ assert "git:unlisted-submodule" in result.stderr
|
|
529 |
+ |
|
530 |
+ |
|
531 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
532 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
533 |
+@pytest.mark.parametrize("fail", ['warn', 'error'])
|
|
534 |
+def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail):
|
|
535 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
536 |
+ |
|
537 |
+ # Make the warning an error if we're testing errors
|
|
538 |
+ if fail == 'error':
|
|
539 |
+ project_template = {
|
|
540 |
+ "name": "foo",
|
|
541 |
+ "fatal-warnings": ['git:unlisted-submodule']
|
|
542 |
+ }
|
|
543 |
+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
|
|
544 |
+ |
|
545 |
+ # Create the submodule first from the 'subrepofiles' subdir
|
|
546 |
+ subrepo = create_repo('git', str(tmpdir), 'subrepo')
|
|
547 |
+ subrepo.create(os.path.join(project, 'subrepofiles'))
|
|
548 |
+ |
|
549 |
+ # Create the repo from 'repofiles' subdir
|
|
550 |
+ repo = create_repo('git', str(tmpdir))
|
|
551 |
+ ref = repo.create(os.path.join(project, 'repofiles'))
|
|
552 |
+ |
|
553 |
+ # Add a submodule pointing to the one we created, but use
|
|
554 |
+ # the original ref, let the submodules appear after tracking
|
|
555 |
+ repo.add_submodule('subdir', 'file://' + subrepo.repo)
|
|
556 |
+ |
|
557 |
+ # Create the source, and delete the explicit configuration
|
|
558 |
+ # of the submodules.
|
|
559 |
+ gitsource = repo.source_config(ref=ref)
|
|
560 |
+ del gitsource['submodules']
|
|
561 |
+ |
|
562 |
+ # Write out our test target
|
|
563 |
+ element = {
|
|
564 |
+ 'kind': 'import',
|
|
565 |
+ 'sources': [
|
|
566 |
+ gitsource
|
|
567 |
+ ]
|
|
568 |
+ }
|
|
569 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
570 |
+ |
|
571 |
+ # Fetch the repo, we will not see the warning because we
|
|
572 |
+ # are still pointing to a ref which predates the submodules
|
|
573 |
+ result = cli.run(project=project, args=['fetch', 'target.bst'])
|
|
574 |
+ result.assert_success()
|
|
575 |
+ assert "git:unlisted-submodule" not in result.stderr
|
|
576 |
+ |
|
577 |
+ # We won't get a warning/error when tracking either, the source
|
|
578 |
+ # has not become Consistency.CACHED so the opportunity to check
|
|
579 |
+ # for the warning has not yet arisen.
|
|
580 |
+ result = cli.run(project=project, args=['track', 'target.bst'])
|
|
581 |
+ result.assert_success()
|
|
582 |
+ assert "git:unlisted-submodule" not in result.stderr
|
|
583 |
+ |
|
584 |
+ # Fetching the repo at the new ref will finally reveal the warning
|
|
585 |
+ result = cli.run(project=project, args=['fetch', 'target.bst'])
|
|
586 |
+ if fail == 'error':
|
|
587 |
+ result.assert_main_error(ErrorDomain.STREAM, None)
|
|
588 |
+ result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
|
|
589 |
+ else:
|
|
590 |
+ result.assert_success()
|
|
591 |
+ assert "git:unlisted-submodule" in result.stderr
|
|
592 |
+ |
|
593 |
+ |
|
594 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
595 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
596 |
+@pytest.mark.parametrize("fail", ['warn', 'error'])
|
|
597 |
+def test_invalid_submodule(cli, tmpdir, datafiles, fail):
|
|
598 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
599 |
+ |
|
600 |
+ # Make the warning an error if we're testing errors
|
|
601 |
+ if fail == 'error':
|
|
602 |
+ project_template = {
|
|
603 |
+ "name": "foo",
|
|
604 |
+ "fatal-warnings": ['git:invalid-submodule']
|
|
605 |
+ }
|
|
606 |
+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
|
|
607 |
+ |
|
608 |
+ # Create the repo from 'repofiles' subdir
|
|
609 |
+ repo = create_repo('git', str(tmpdir))
|
|
610 |
+ ref = repo.create(os.path.join(project, 'repofiles'))
|
|
611 |
+ |
|
612 |
+ # Create the source without any submodules, and add
|
|
613 |
+ # an invalid submodule configuration to it.
|
|
614 |
+ #
|
|
615 |
+ # We expect this to cause an invalid submodule warning
|
|
616 |
+ # after the source has been fetched and we know what
|
|
617 |
+ # the real submodules actually are.
|
|
618 |
+ #
|
|
619 |
+ gitsource = repo.source_config(ref=ref)
|
|
620 |
+ gitsource['submodules'] = {
|
|
621 |
+ 'subdir': {
|
|
622 |
+ 'url': 'https://pony.org/repo.git'
|
|
623 |
+ }
|
|
624 |
+ }
|
|
625 |
+ |
|
626 |
+ # Write out our test target
|
|
627 |
+ element = {
|
|
628 |
+ 'kind': 'import',
|
|
629 |
+ 'sources': [
|
|
630 |
+ gitsource
|
|
631 |
+ ]
|
|
632 |
+ }
|
|
633 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
634 |
+ |
|
635 |
+ # We will not see the warning or error before the first fetch, because
|
|
636 |
+ # we don't have the repository yet and so we have no knowledge of
|
|
637 |
+ # the unlisted submodule.
|
|
638 |
+ result = cli.run(project=project, args=['show', 'target.bst'])
|
|
639 |
+ result.assert_success()
|
|
640 |
+ assert "git:invalid-submodule" not in result.stderr
|
|
641 |
+ |
|
642 |
+ # We will notice this directly in fetch, as it will try to fetch
|
|
643 |
+ # the submodules it discovers as a result of fetching the primary repo.
|
|
644 |
+ result = cli.run(project=project, args=['fetch', 'target.bst'])
|
|
645 |
+ |
|
646 |
+ # Assert a warning or an error depending on what we're checking
|
|
647 |
+ if fail == 'error':
|
|
648 |
+ result.assert_main_error(ErrorDomain.STREAM, None)
|
|
649 |
+ result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
|
|
650 |
+ else:
|
|
651 |
+ result.assert_success()
|
|
652 |
+ assert "git:invalid-submodule" in result.stderr
|
|
653 |
+ |
|
654 |
+ # Now that we've fetched it, `bst show` will discover the unlisted submodule too
|
|
655 |
+ result = cli.run(project=project, args=['show', 'target.bst'])
|
|
656 |
+ |
|
657 |
+ # Assert a warning or an error depending on what we're checking
|
|
658 |
+ if fail == 'error':
|
|
659 |
+ result.assert_main_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
|
|
660 |
+ else:
|
|
661 |
+ result.assert_success()
|
|
662 |
+ assert "git:invalid-submodule" in result.stderr
|
|
663 |
+ |
|
664 |
+ |
|
665 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
666 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
667 |
+@pytest.mark.parametrize("fail", ['warn', 'error'])
|
|
668 |
+def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
|
|
669 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
670 |
+ |
|
671 |
+ # Make the warning an error if we're testing errors
|
|
672 |
+ if fail == 'error':
|
|
673 |
+ project_template = {
|
|
674 |
+ "name": "foo",
|
|
675 |
+ "fatal-warnings": ['git:invalid-submodule']
|
|
676 |
+ }
|
|
677 |
+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
|
|
678 |
+ |
|
679 |
+ # Create the submodule first from the 'subrepofiles' subdir
|
|
680 |
+ subrepo = create_repo('git', str(tmpdir), 'subrepo')
|
|
681 |
+ subrepo.create(os.path.join(project, 'subrepofiles'))
|
|
682 |
+ |
|
683 |
+ # Create the repo from 'repofiles' subdir
|
|
684 |
+ repo = create_repo('git', str(tmpdir))
|
|
685 |
+ ref = repo.create(os.path.join(project, 'repofiles'))
|
|
686 |
+ |
|
687 |
+ # Add a submodule pointing to the one we created
|
|
688 |
+ ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
|
|
689 |
+ |
|
690 |
+ # Add a commit beyond the ref which *removes* the submodule we've added
|
|
691 |
+ repo.remove_path('subdir')
|
|
692 |
+ |
|
693 |
+ # Create the source, this will keep the submodules so initially
|
|
694 |
+ # the configuration is valid for the ref we're using
|
|
695 |
+ gitsource = repo.source_config(ref=ref)
|
|
696 |
+ |
|
697 |
+ # Write out our test target
|
|
698 |
+ element = {
|
|
699 |
+ 'kind': 'import',
|
|
700 |
+ 'sources': [
|
|
701 |
+ gitsource
|
|
702 |
+ ]
|
|
703 |
+ }
|
|
704 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
705 |
+ |
|
706 |
+ # Fetch the repo, we will not see the warning because we
|
|
707 |
+ # are still pointing to a ref which predates the submodules
|
|
708 |
+ result = cli.run(project=project, args=['fetch', 'target.bst'])
|
|
709 |
+ result.assert_success()
|
|
710 |
+ assert "git:invalid-submodule" not in result.stderr
|
|
711 |
+ |
|
712 |
+ # In this case, we will get the error directly after tracking,
|
|
713 |
+ # since the new HEAD does not require any submodules which are
|
|
714 |
+ # not locally cached, the Source will be CACHED directly after
|
|
715 |
+ # tracking and the validations will occur as a result.
|
|
716 |
+ #
|
|
717 |
+ result = cli.run(project=project, args=['track', 'target.bst'])
|
|
718 |
+ if fail == 'error':
|
|
719 |
+ result.assert_main_error(ErrorDomain.STREAM, None)
|
|
720 |
+ result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
|
|
721 |
+ else:
|
|
722 |
+ result.assert_success()
|
|
723 |
+ assert "git:invalid-submodule" in result.stderr
|
|
724 |
+ |
|
725 |
+ |
|
458 | 726 |
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
459 | 727 |
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
460 | 728 |
@pytest.mark.parametrize("ref_format", ['sha1', 'git-describe'])
|
... | ... | @@ -76,6 +76,12 @@ class Git(Repo): |
76 | 76 |
self._run_git('commit', '-m', 'Added the submodule')
|
77 | 77 |
return self.latest_commit()
|
78 | 78 |
|
79 |
+ # This can also be used to a file or a submodule
|
|
80 |
+ def remove_path(self, path):
|
|
81 |
+ self._run_git('rm', path)
|
|
82 |
+ self._run_git('commit', '-m', 'Removing {}'.format(path))
|
|
83 |
+ return self.latest_commit()
|
|
84 |
+ |
|
79 | 85 |
def source_config(self, ref=None, checkout_submodules=None):
|
80 | 86 |
config = {
|
81 | 87 |
'kind': 'git',
|