[Notes] [Git][BuildStream/buildstream][valentindavid/fallback_mirror_ostree-1.2] 3 commits: examples: Switch ftp.gnu.org to mirror

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/fallback_mirror_ostree-1.2 at BuildStream / buildstream

Commits:

• 0795a4c5
  by Josh Smith at 2018-08-15T14:34:04Z

  examples: Switch ftp.gnu.org to mirror
  
  This should avoid current issues accessing the necessary sources which
  are blocking CI currently.
  

• ac97bc5e
  by Valentin David at 2018-08-15T15:25:53Z

  Merge branch 'Qinusty/gnu-mirror-backport-1.2' into 'bst-1.2'
  
  Qinusty/gnu mirror backport 1.2
  
  See merge request BuildStream/buildstream!668

• 0754810f
  by Valentin David at 2018-08-15T15:37:41Z

  Fix ostree repository mirroring
  
  Ostree mirrors were not sharing the same local repository, so it was
  impossible the request refs from the right local repository when data
  was fetched from a mirror rather than upstream.
  
  Instead of having several repository with one remote each, we now
  have one repository with several remotes.
  
  This fixes #538.
  

3 changed files:

• buildstream/plugins/sources/ostree.py
• doc/examples/autotools/project.conf
• tests/frontend/mirror.py

Changes:

buildstream/plugins/sources/ostree.py

---

...	...	@@ -71,7 +71,7 @@ class OSTreeSource(Source):
71	71	self.ref = self.node_get_member(node, str, 'ref', None)
72	72	self.tracking = self.node_get_member(node, str, 'track', None)
73	73	self.mirror = os.path.join(self.get_mirror_directory(),
74		- utils.url_directory_name(self.url))
	74	+ utils.url_directory_name(self.original_url))
75	75
76	76	# (optional) Not all repos are signed. But if they are, get the gpg key
77	77	self.gpg_key_path = None
...	...	@@ -104,10 +104,11 @@ class OSTreeSource(Source):
104	104	return None
105	105
106	106	self.ensure()
	107	+ remote_name = self.ensure_remote(self.url)
107	108	with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
108	109	.format(self.tracking, self.url)):
109	110	try:
110		- _ostree.fetch(self.repo, ref=self.tracking, progress=self.progress)
	111	+ _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
111	112	except OSTreeError as e:
112	113	raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
113	114	.format(self, self.tracking, self.url, e)) from e
...	...	@@ -116,11 +117,12 @@ class OSTreeSource(Source):
116	117
117	118	def fetch(self):
118	119	self.ensure()
	120	+ remote_name = self.ensure_remote(self.url)
119	121	if not _ostree.exists(self.repo, self.ref):
120	122	with self.timed_activity("Fetching remote ref: {} from origin: {}"
121	123	.format(self.ref, self.url)):
122	124	try:
123		- _ostree.fetch(self.repo, ref=self.ref, progress=self.progress)
	125	+ _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
124	126	except OSTreeError as e:
125	127	raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
126	128	.format(self, self.ref, self.url, e)) from e
...	...	@@ -171,14 +173,22 @@ class OSTreeSource(Source):
171	173	self.status("Creating local mirror for {}".format(self.url))
172	174
173	175	self.repo = _ostree.ensure(self.mirror, True)
174		- gpg_key = None
175		- if self.gpg_key_path:
176		- gpg_key = 'file://' + self.gpg_key_path
177	176
178		- try:
179		- _ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key)
180		- except OSTreeError as e:
181		- raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
	177	+ def ensure_remote(self, url):
	178	+ if self.original_url == self.url:
	179	+ remote_name = 'origin'
	180	+ else:
	181	+ remote_name = utils.url_directory_name(url)
	182	+
	183	+ gpg_key = None
	184	+ if self.gpg_key_path:
	185	+ gpg_key = 'file://' + self.gpg_key_path
	186	+
	187	+ try:
	188	+ _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
	189	+ except OSTreeError as e:
	190	+ raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
	191	+ return remote_name
182	192
183	193	def progress(self, percent, message):
184	194	self.status(message)

doc/examples/autotools/project.conf

---

...	...	@@ -10,4 +10,4 @@ element-path: elements
10	10	# Define some aliases for the tarballs we download
11	11	aliases:
12	12	alpine: https://gnome7.codethink.co.uk/tarballs/
13		- gnu: https://ftp.gnu.org/gnu/automake/
	13	+ gnu: https://ftpmirror.gnu.org/gnu/automake/

tests/frontend/mirror.py

---

...	...	@@ -405,14 +405,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
405	405	@pytest.mark.datafiles(DATA_DIR)
406	406	@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
407	407	def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
408		- if kind == 'git':
409		- # FIXME: Mirroring fallback does not work with git because it tries to
410		- # fetch submodules on upstream.
411		- pytest.skip("Bug #537 - Mirror fallback does not work for git")
412		- if kind == 'ostree':
413		- # FIXME: Mirroring fallback fails with ostree
414		- pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
415		-
416	408	bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
417	409	upstream_repodir = os.path.join(str(tmpdir), 'upstream')
418	410	mirror_repodir = os.path.join(str(tmpdir), 'mirror')

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.