[openwrt/openwrt] download: add support for gitweb snapshots

[LEDE Commits]

robimarko pushed a commit to openwrt/openwrt.git, branch main:
https://git.openwrt.org/685fc753a7fd0e87cf61204620348ec0b40ba426

commit 685fc753a7fd0e87cf61204620348ec0b40ba426
Author: Michael Pratt <mcpratt at pm.me>
AuthorDate: Sat May 31 14:00:45 2025 -0400

    download: add support for gitweb snapshots
    
    When downloading a snapshot archive from gitweb,
    the filename is not part of the URL,
    and adding the filename to the URL causes errors.
    
    The gitweb API exclusively uses query parameters
    instead of paths in order to execute snapshot downloads.
    
    Add a condition to the Perl download script
    that removes the filename if the relevant
    query parameter matches in the URL.
    
    Also, to reduce server load of the original sources
    try the Openwrt CDN servers first for these downloads.
    
    Even though snapshot downloads are not ideal
    due to the impact on the source's server health,
    they are better for download performance than using git only.
    Therefore, attempting it last will reduce the impact
    and thus encourage maintainers to keep the option enabled.
    
    This change is partly inspired by a conversation linked below
    about snapshot downloads and server performance issues
    which led to the feature being disabled for a particular server.
    
    Link: https://lists.gnu.org/archive/html/bug-gnulib/2024-12/msg00124.html
    Signed-off-by: Michael Pratt <mcpratt at pm.me>
    Link: https://github.com/openwrt/openwrt/pull/16522
    Signed-off-by: Robert Marko <robimarko at gmail.com>
---
 scripts/download.pl | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/scripts/download.pl b/scripts/download.pl
index c6c9b8e56c..09dc91b04b 100755
--- a/scripts/download.pl
+++ b/scripts/download.pl
@@ -163,6 +163,7 @@ sub download
 	my $mirror = shift;
 	my $download_filename = shift;
 	my @additional_mirrors = @_;
+	my @cmd;
 
 	$mirror =~ s!/$!!;
 
@@ -209,7 +210,11 @@ sub download
 			}
 		};
 	} else {
-		my @cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors);
+		if ($mirror =~ /a=snapshot/) {
+			@cmd = download_cmd("$mirror", $download_filename, @additional_mirrors);
+		} else {
+			@cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors);
+		}
 		print STDERR "+ ".join(" ", at cmd)."\n";
 		open(FETCH_FD, '-|', @cmd) or die "Cannot launch aria2c, curl or wget.\n";
 		$hash_cmd and do {
@@ -317,14 +322,23 @@ if (-f "$target/$filename") {
 
 $download_tool = select_tool();
 
+my $mirror = shift @mirrors;
+
+# Try snapshot original source last
+if ($mirror =~ /snapshot/) {
+	push @mirrors, $mirror;
+	$mirror = shift @mirrors;
+}
+
 while (!-f "$target/$filename") {
-	my $mirror = shift @mirrors;
 	$mirror or die "No more mirrors to try - giving up.\n";
 
 	download($mirror, $url_filename, @mirrors);
 	if (!-f "$target/$filename" && $url_filename ne $filename) {
 		download($mirror, $filename, @mirrors);
 	}
+
+	$mirror = shift @mirrors;
 }
 
 $SIG{INT} = \&cleanup;