Skip to content

fix(pypi): propagate fails if overrides are passed only one index is used #3666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aignas wants to merge 1 commit into
base: main
Choose a base branch
from
+92 −13
Open

fix(pypi): propagate fails if overrides are passed only one index is used #3666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ END_UNRELEASED_TEMPLATE
Other changes:
* (pypi) Update dependencies used for `compile_pip_requirements`, building
sdists in the `whl_library` rule and fetching wheels using `pip`.
* (pypi) We will set `allow_fail` to `False` if the {attr}`experimental_index_url_overrides` is set
to a non-empty value. This means that failures will be no-longer cached in this particular case.
([#3260](https://github.com/bazel-contrib/rules_python/issues/3260) and
[#2632](https://github.com/bazel-contrib/rules_python/issues/2632))

{#v0-0-0-fixed}
### Fixed
Expand Down
14 changes: 9 additions & 5 deletions python/private/pypi/simpleapi_download.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,21 @@ def simpleapi_download(
for p, i in (attr.index_url_overrides or {}).items()
}

download_kwargs = {}
if bazel_features.external_deps.download_has_block_param:
download_kwargs["block"] = not parallel_download

# NOTE @aignas 2024-03-31: we are not merging results from multiple indexes
# to replicate how `pip` would handle this case.
contents = {}
index_urls = [attr.index_url] + attr.extra_index_urls
read_simpleapi = read_simpleapi or _read_simpleapi

download_kwargs = {}
if bazel_features.external_deps.download_has_block_param:
download_kwargs["block"] = not parallel_download

if len(index_urls) == 1 or index_url_overrides:
download_kwargs["allow_fail"] = False
else:
download_kwargs["allow_fail"] = True
Comment on lines +84 to +87
Copy link
Contributor

@gemini-code-assist gemini-code-assist bot Mar 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The current logic for setting allow_fail has an unintended side effect when both index_url_overrides and extra_index_urls are used. When index_url_overrides is set, allow_fail is globally set to False. This prevents the intended fallback mechanism for packages that are not in the override map but might exist in one of the extra_index_urls.

For example, consider this scenario:

  • index_url = "A"
  • extra_index_urls = ["B"]
  • index_url_overrides = {"pkg1": "C"}
  • requirements.txt contains pkg1 and pkg2.
  • pkg2 exists in index "B" but not "A".

With the current change, the download for pkg2 from index "A" will fail, and because allow_fail is False, the entire process will stop. It will not attempt to download pkg2 from index "B". This seems to break the expected fallback behavior.

The allow_fail logic should probably be more granular, determined on a per-package basis. It should be False only when a download is expected to be definitive (e.g., only one index URL to try for a package, or for a package with a specific override).

The existing tests do not seem to cover this specific combination of multiple indexes and an override map, which is why this issue might have been missed.


input_sources = attr.sources

found_on_index = {}
Expand Down Expand Up @@ -225,7 +230,6 @@ def _read_simpleapi(ctx, url, attr, cache, versions, get_auth = None, **download
url = [real_url],
output = output,
auth = get_auth(ctx, [real_url], ctx_attr = attr),
allow_fail = True,
**download_kwargs
)

Expand Down
87 changes: 79 additions & 8 deletions tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@ _tests = []
def _test_simple(env):
calls = []

def read_simpleapi(ctx, url, versions, attr, cache, get_auth, block):
def read_simpleapi(ctx, url, versions, attr, cache, get_auth, block, allow_fail):
_ = ctx # buildifier: disable=unused-variable
_ = attr
_ = cache
_ = get_auth
_ = versions
env.expect.that_bool(block).equals(False)
env.expect.that_bool(allow_fail).equals(True)
calls.append(url)
if "foo" in url and "main" in url:
return struct(
Expand Down Expand Up @@ -96,13 +97,14 @@ def _test_fail(env):
calls = []
fails = []

def read_simpleapi(ctx, url, versions, attr, cache, get_auth, block):
def read_simpleapi(ctx, url, versions, attr, cache, get_auth, block, allow_fail):
_ = ctx # buildifier: disable=unused-variable
_ = attr
_ = cache
_ = get_auth
_ = versions
env.expect.that_bool(block).equals(False)
env.expect.that_bool(allow_fail).equals(True)
calls.append(url)
if "foo" in url:
return struct(
Expand Down Expand Up @@ -130,9 +132,7 @@ def _test_fail(env):
report_progress = lambda _: None,
),
attr = struct(
index_url_overrides = {
"foo": "invalid",
},
index_url_overrides = {},
index_url = "main",
extra_index_urls = ["extra"],
sources = {"bar": None, "baz": None, "foo": None},
Expand All @@ -149,7 +149,7 @@ def _test_fail(env):
Failed to download metadata of the following packages from urls:
{
"bar": ["main", "extra"],
"foo": "invalid",
"foo": ["main", "extra"],
}

If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=[
Expand All @@ -159,15 +159,86 @@ If you would like to skip downloading metadata for these packages please add 'si
""",
])
env.expect.that_collection(calls).contains_exactly([
"invalid/foo/",
"main/foo/",
"main/bar/",
"main/baz/",
"invalid/foo/",
"extra/foo/",
"extra/bar/",
])

_tests.append(_test_fail)

def _test_allow_fail_single_index(env):
calls = []
fails = []

def read_simpleapi(ctx, *, url, versions, attr, cache, get_auth, block, allow_fail):
_ = ctx # buildifier: disable=unused-variable
Copy link
Collaborator

@rickeylev rickeylev Mar 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: assign them all to underscore in a single line. Just saves a bit of vertical space

_ = attr
_ = cache
_ = get_auth
_ = versions
env.expect.that_bool(block).equals(False)
env.expect.that_bool(allow_fail).equals(False)
calls.append(url)
return struct(
output = struct(
sdists = {"deadbeef": url.strip("/").split("/")[-1]},
whls = {"deadb33f": url.strip("/").split("/")[-1]},
sha256s_by_version = {"fizz": url.strip("/").split("/")[-1]},
),
success = True,
)

contents = simpleapi_download(
ctx = struct(
getenv = {}.get,
report_progress = lambda _: None,
),
attr = struct(
index_url_overrides = {
"foo": "extra",
},
index_url = "main",
extra_index_urls = [],
sources = {"bar": None, "baz": None, "foo": None},
envsubst = [],
),
cache = pypi_cache(),
parallel_download = True,
read_simpleapi = read_simpleapi,
_fail = fails.append,
)

env.expect.that_collection(fails).contains_exactly([])
env.expect.that_collection(calls).contains_exactly([
"main/bar/",
"main/baz/",
"extra/foo/",
])
env.expect.that_dict(contents).contains_exactly({
"bar": struct(
index_url = "main/bar/",
sdists = {"deadbeef": "bar"},
sha256s_by_version = {"fizz": "bar"},
whls = {"deadb33f": "bar"},
),
"baz": struct(
index_url = "main/baz/",
sdists = {"deadbeef": "baz"},
sha256s_by_version = {"fizz": "baz"},
whls = {"deadb33f": "baz"},
),
"foo": struct(
index_url = "extra/foo/",
sdists = {"deadbeef": "foo"},
sha256s_by_version = {"fizz": "foo"},
whls = {"deadb33f": "foo"},
),
})

_tests.append(_test_allow_fail_single_index)

def _test_download_url(env):
downloads = {}

Expand Down