| ... |
... |
@@ -215,9 +215,11 @@ class Source(Plugin): |
|
215
|
215
|
When set to True:
|
|
216
|
216
|
* all sources listed before this source in the given element will be
|
|
217
|
217
|
fetched before this source is tracked
|
|
218
|
|
- * Source.track() will be called with an additional keywork argument
|
|
219
|
|
- `previous_sources_dir` that will contain a list of sources
|
|
|
218
|
+ * Source.track() will be called with an additional keyword argument
|
|
|
219
|
+ `previous_sources_dir` where previous sources will be staged
|
|
220
|
220
|
* this source can not be the first source for an element
|
|
|
221
|
+
|
|
|
222
|
+ *Since: 1.4*
|
|
221
|
223
|
"""
|
|
222
|
224
|
|
|
223
|
225
|
requires_previous_sources_fetch = False
|
| ... |
... |
@@ -227,8 +229,10 @@ class Source(Plugin): |
|
227
|
229
|
* all sources listed before this source in the given element will be
|
|
228
|
230
|
fetched before this source is fetched
|
|
229
|
231
|
* Source.fetch() will be called with an additional keyword argument
|
|
230
|
|
- `previous_sources_dir` that will contain a list of sources
|
|
|
232
|
+ `previous_sources_dir` where previous sources will be staged
|
|
231
|
233
|
* this source can not be the first source for an element
|
|
|
234
|
+
|
|
|
235
|
+ *Since: 1.4*
|
|
232
|
236
|
"""
|
|
233
|
237
|
|
|
234
|
238
|
def __init__(self, context, project, meta, *, alias_override=None):
|
| ... |
... |
@@ -545,59 +549,15 @@ class Source(Plugin): |
|
545
|
549
|
# previous_sources (list): List of Sources listed prior to this source
|
|
546
|
550
|
#
|
|
547
|
551
|
def _fetch(self, previous_sources):
|
|
548
|
|
- def __fetch(**kwargs):
|
|
549
|
|
- project = self._get_project()
|
|
550
|
|
- source_fetchers = self.get_source_fetchers()
|
|
551
|
|
- if source_fetchers:
|
|
552
|
|
- for fetcher in source_fetchers:
|
|
553
|
|
- alias = fetcher._get_alias()
|
|
554
|
|
- success = False
|
|
555
|
|
- for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
556
|
|
- try:
|
|
557
|
|
- fetcher.fetch(uri)
|
|
558
|
|
- # FIXME: Need to consider temporary vs. permanent failures,
|
|
559
|
|
- # and how this works with retries.
|
|
560
|
|
- except BstError as e:
|
|
561
|
|
- last_error = e
|
|
562
|
|
- continue
|
|
563
|
|
- success = True
|
|
564
|
|
- break
|
|
565
|
|
- if not success:
|
|
566
|
|
- raise last_error
|
|
567
|
|
- else:
|
|
568
|
|
- alias = self._get_alias()
|
|
569
|
|
- if self.__first_pass:
|
|
570
|
|
- mirrors = project.first_pass_config.mirrors
|
|
571
|
|
- else:
|
|
572
|
|
- mirrors = project.config.mirrors
|
|
573
|
|
- if not mirrors or not alias:
|
|
574
|
|
- self.fetch(**kwargs)
|
|
575
|
|
- return
|
|
576
|
|
-
|
|
577
|
|
- context = self._get_context()
|
|
578
|
|
- source_kind = type(self)
|
|
579
|
|
- for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
580
|
|
- new_source = source_kind(context, project, self.__meta,
|
|
581
|
|
- alias_override=(alias, uri))
|
|
582
|
|
- new_source._preflight()
|
|
583
|
|
- try:
|
|
584
|
|
- new_source.fetch(**kwargs)
|
|
585
|
|
- # FIXME: Need to consider temporary vs. permanent failures,
|
|
586
|
|
- # and how this works with retries.
|
|
587
|
|
- except BstError as e:
|
|
588
|
|
- last_error = e
|
|
589
|
|
- continue
|
|
590
|
|
- return
|
|
591
|
|
- raise last_error
|
|
592
|
552
|
|
|
593
|
553
|
if self.requires_previous_sources_fetch:
|
|
594
|
554
|
self.__ensure_previous_sources(previous_sources)
|
|
595
|
555
|
with self.tempdir() as staging_directory:
|
|
596
|
556
|
for src in previous_sources:
|
|
597
|
557
|
src._stage(staging_directory)
|
|
598
|
|
- __fetch(previous_sources_dir=staging_directory)
|
|
|
558
|
+ self.__do_fetch(previous_sources_dir=staging_directory)
|
|
599
|
559
|
else:
|
|
600
|
|
- __fetch()
|
|
|
560
|
+ self.__do_fetch()
|
|
601
|
561
|
|
|
602
|
562
|
# Wrapper for stage() api which gives the source
|
|
603
|
563
|
# plugin a fully constructed path considering the
|
| ... |
... |
@@ -832,16 +792,15 @@ class Source(Plugin): |
|
832
|
792
|
|
|
833
|
793
|
return new_ref
|
|
834
|
794
|
|
|
835
|
|
- # _is_transform()
|
|
|
795
|
+ # _requires_previous_sources()
|
|
836
|
796
|
#
|
|
837
|
|
- # A plugin is considered a source transform plugin if it requires access to
|
|
838
|
|
- # previous sources for its tracking or fetching. Such sources cannot be the
|
|
839
|
|
- # first source of any element.
|
|
|
797
|
+ # If a plugin requires access to previous sources at track or fetch time,
|
|
|
798
|
+ # then it cannot be the first source of an elemenet.
|
|
840
|
799
|
#
|
|
841
|
800
|
# Returns:
|
|
842
|
|
- # (bool): Whetner this is a source transform plugin.
|
|
|
801
|
+ # (bool): Whether this source requires access to previous sources
|
|
843
|
802
|
#
|
|
844
|
|
- def _is_transform(self):
|
|
|
803
|
+ def _requires_previous_sources(self):
|
|
845
|
804
|
return self.requires_previous_sources_track or self.requires_previous_sources_fetch
|
|
846
|
805
|
|
|
847
|
806
|
# Returns the alias if it's defined in the project
|
| ... |
... |
@@ -859,6 +818,52 @@ class Source(Plugin): |
|
859
|
818
|
# Local Private Methods #
|
|
860
|
819
|
#############################################################
|
|
861
|
820
|
|
|
|
821
|
+ # Tries to call fetch for every mirror, stopping once it succeeds
|
|
|
822
|
+ def __do_fetch(self, **kwargs):
|
|
|
823
|
+ project = self._get_project()
|
|
|
824
|
+ source_fetchers = self.get_source_fetchers()
|
|
|
825
|
+ if source_fetchers:
|
|
|
826
|
+ for fetcher in source_fetchers:
|
|
|
827
|
+ alias = fetcher._get_alias()
|
|
|
828
|
+ success = False
|
|
|
829
|
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
|
830
|
+ try:
|
|
|
831
|
+ fetcher.fetch(uri)
|
|
|
832
|
+ # FIXME: Need to consider temporary vs. permanent failures,
|
|
|
833
|
+ # and how this works with retries.
|
|
|
834
|
+ except BstError as e:
|
|
|
835
|
+ last_error = e
|
|
|
836
|
+ continue
|
|
|
837
|
+ success = True
|
|
|
838
|
+ break
|
|
|
839
|
+ if not success:
|
|
|
840
|
+ raise last_error
|
|
|
841
|
+ else:
|
|
|
842
|
+ alias = self._get_alias()
|
|
|
843
|
+ if self.__first_pass:
|
|
|
844
|
+ mirrors = project.first_pass_config.mirrors
|
|
|
845
|
+ else:
|
|
|
846
|
+ mirrors = project.config.mirrors
|
|
|
847
|
+ if not mirrors or not alias:
|
|
|
848
|
+ self.fetch(**kwargs)
|
|
|
849
|
+ return
|
|
|
850
|
+
|
|
|
851
|
+ context = self._get_context()
|
|
|
852
|
+ source_kind = type(self)
|
|
|
853
|
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
|
854
|
+ new_source = source_kind(context, project, self.__meta,
|
|
|
855
|
+ alias_override=(alias, uri))
|
|
|
856
|
+ new_source._preflight()
|
|
|
857
|
+ try:
|
|
|
858
|
+ new_source.fetch(**kwargs)
|
|
|
859
|
+ # FIXME: Need to consider temporary vs. permanent failures,
|
|
|
860
|
+ # and how this works with retries.
|
|
|
861
|
+ except BstError as e:
|
|
|
862
|
+ last_error = e
|
|
|
863
|
+ continue
|
|
|
864
|
+ return
|
|
|
865
|
+ raise last_error
|
|
|
866
|
+
|
|
862
|
867
|
# Tries to call track for every mirror, stopping once it succeeds
|
|
863
|
868
|
def __do_track(self, **kwargs):
|
|
864
|
869
|
project = self._get_project()
|
