yuzu/vcpkg
1
0
Fork 0
mirror of https://github.com/microsoft/vcpkg synced 2024-11-21 16:09:03 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[vcpkg] Add experimental $X_VCPKG_ASSET_SOURCES for source caching (#13639)

Browse source 
* [vcpkg] Add experimental $VCPKG_X_READWRITE_MIRROR_URL_TEMPLATE for source caching

* [vcpkg] Use prerelease version for testing

* [docs] Add assetcaching.md

* Revert bootstrap changes

* [vcpkg_download_distfile] Deprecate SILENT_EXIT

* cmake_parse_arguments() always defines option arguments

* Adjust CI baseline

* [vcpkg_download_distfile] Address CR comments, restore ARIA2

* [CI] Add source caching to Linux and OSX CI passes

* [ci.baseline.txt] Skip chartdir on more platforms
This commit is contained in:
ras0219 2021-06-04 16:48:26 -07:00 committed by GitHub
parent 834a87732c
commit 8c497bb647
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 127 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
docs/users/assetcaching.md Normal file
View file

@ -0,0 +1,43 @@
# Asset Caching
**The latest version of this documentation is available on [GitHub](https://github.com/Microsoft/vcpkg/tree/master/docs/users/assetcaching.md).**
**Experimental feature: this may change or be removed at any time**
Vcpkg can utilize mirrors to cache downloaded assets, ensuring continued operation even if the original source changes
or disappears.
In-tool help is available via `vcpkg help assetcaching`.
## Configuration
Asset caching can be configured by setting the environment variable `X_VCPKG_ASSET_SOURCES` to a semicolon-delimited
list of source strings. Characters can be escaped using backtick (\`).
### Valid source strings
The `<rw>` optional parameter for certain strings controls how they will be accessed. It can be specified as `read`,
`write`, or `readwrite` and defaults to `read`.
#### `clear`
Syntax: `clear`
Removes all previous sources
#### `x-azurl`
Syntax: `x-azurl,<url>[,<sas>[,<rw>]]`
Adds an Azure Blob Storage source, optionally using Shared Access Signature validation. URL should include the container
path and be terminated with a trailing `/`. SAS, if defined, should be prefixed with a `?`. Non-Azure servers will also
work if they respond to GET and PUT requests of the form: `<url><sha512><sas>`.
See also the [binary caching documentation for Azure Blob Storage](binarycaching.md#azure-blob-storage-experimental) for
more information on how to set up an `x-azurl` source.
#### `x-block-origin`
Syntax: `x-block-origin`
Disables use of the original URLs in case the mirror does not have the file available.

6
docs/users/config-environment.md
View file

@ -87,3 +87,9 @@ This environment variable changes the metadata of produced NuGet packages. See [
#### VCPKG_USE_NUGET_CACHE
This environment variable allows using NuGet's cache for every nuget-based binary source. See [Binary Caching](binarycaching.md#NuGets-cache) for more details.
#### X_VCPKG_ASSET_SOURCES
> Note: This is an experimental feature and may change or be removed at any time
This environment variable allows using a private mirror for all SHA512-tagged assets. See [Asset Caching](assetcaching.md) for more details.

3
scripts/azure-pipelines/linux/azure-pipelines.yml
View file

@ -14,6 +14,9 @@ jobs:
value: /mnt/vcpkg-ci
- name: VCPKG_DOWNLOADS
value: /mnt/vcpkg-ci/downloads
- group: azblob-test-sas-group
- name: X_VCPKG_ASSET_SOURCES
value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"
steps:
- bash: df -h

2
scripts/azure-pipelines/osx/azure-pipelines.yml
View file

@ -17,6 +17,8 @@ jobs:
- group: osx-2021-04-16
- name: BINARY_SOURCE_STUB
value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"
- name: X_VCPKG_ASSET_SOURCES
value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"
steps:
- bash: df -h

3
scripts/azure-pipelines/windows/azure-pipelines.yml
View file

@ -14,6 +14,9 @@ jobs:
value: D:\
- name: VCPKG_DOWNLOADS
value: D:\downloads
- group: azblob-test-sas-group
- name: X_VCPKG_ASSET_SOURCES
value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"
steps:
- task: PowerShell@2

8
scripts/ci.baseline.txt
View file

@ -106,10 +106,16 @@ charls:x64-windows=skip
charls:x64-windows-static=skip
charls:x64-windows-static-md=skip
charls:x86-windows=skip
# chartdir does not offer stable download URLs
chartdir:arm64-windows=fail
chartdir:arm-uwp=fail
chartdir:x64-uwp=fail
chartdir:x64-windows-static=fail
chartdir:x86-windows=fail
chartdir:x64-windows=fail
chartdir:x64-windows-static-md=fail
chartdir:x64-linux=fail
chartdir:x64-osx=fail
chmlib:arm-uwp=fail
chmlib:x64-uwp=fail
# Chromium Base requires a recent version of Clang to be installed.
@ -1625,6 +1631,8 @@ v8:arm64-windows=fail
v8:arm-uwp=fail
v8:x64-osx=fail
v8:x64-uwp=fail
# The domain hosting vamp-sdk uses a newer root cert than is available in our linux CI
vamp-sdk:x64-linux=fail
vectorclass:arm64-windows=fail
vectorclass:arm-uwp=fail
vlpp:x64-osx=fail

79
scripts/cmake/vcpkg_download_distfile.cmake
View file

@ -67,7 +67,12 @@ function(vcpkg_download_distfile VAR)
if(NOT DEFINED vcpkg_download_distfile_FILENAME)
message(FATAL_ERROR "vcpkg_download_distfile requires a FILENAME argument.")
endif()
if(NOT _VCPKG_INTERNAL_NO_HASH_CHECK)
if(vcpkg_download_distfile_SILENT_EXIT)
message(WARNING "SILENT_EXIT has been deprecated as an argument to vcpkg_download_distfile -- remove the argument to resolve this warning")
endif()
if(_VCPKG_INTERNAL_NO_HASH_CHECK)
set(vcpkg_download_distfile_SKIP_SHA512 1)
else()
if(NOT vcpkg_download_distfile_SKIP_SHA512 AND NOT DEFINED vcpkg_download_distfile_SHA512)
message(FATAL_ERROR "vcpkg_download_distfile requires a SHA512 argument. If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.")
endif()
@ -113,7 +118,7 @@ function(vcpkg_download_distfile VAR)
if(EXISTS "${downloaded_file_path}")
if(NOT vcpkg_download_distfile_QUIET)
message(STATUS "Using cached ${downloaded_file_path}")
message(STATUS "Using ${downloaded_file_path}")
endif()
test_hash("${downloaded_file_path}" "cached file" "Please delete the file and retry if this file should be downloaded again.")
else()
@ -127,7 +132,7 @@ function(vcpkg_download_distfile VAR)
vcpkg_find_acquire_program("ARIA2")
message(STATUS "Downloading ${vcpkg_download_distfile_FILENAME}...")
if(vcpkg_download_distfile_HEADERS)
foreach(header ${vcpkg_download_distfile_HEADERS})
foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
list(APPEND request_headers "--header=${header}")
endforeach()
endif()
@ -159,29 +164,60 @@ function(vcpkg_download_distfile VAR)
)
set(download_success 1)
endif()
else()
elseif(vcpkg_download_distfile_SKIP_SHA512)
set(download_success 0)
set(request_headers)
if(vcpkg_download_distfile_HEADERS)
foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
list(APPEND request_headers HTTPHEADER ${header})
endforeach()
endif()
foreach(url IN LISTS vcpkg_download_distfile_URLS)
message(STATUS "Downloading ${url} -> ${vcpkg_download_distfile_FILENAME}...")
if(vcpkg_download_distfile_HEADERS)
foreach(header ${vcpkg_download_distfile_HEADERS})
list(APPEND request_headers HTTPHEADER ${header})
endforeach()
endif()
file(DOWNLOAD "${url}" "${download_file_path_part}" STATUS download_status ${request_headers})
list(GET download_status 0 status_code)
if (NOT "${status_code}" STREQUAL "0")
message(STATUS "Downloading ${url}... Failed. Status: ${download_status}")
set(download_success 0)
else()
test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
file(MAKE_DIRECTORY "${downloaded_file_dir}")
file(RENAME ${download_file_path_part} ${downloaded_file_path})
set(download_success 1)
break()
endif()
endforeach(url)
else()
set(urls)
foreach(url IN LISTS vcpkg_download_distfile_URLS)
list(APPEND urls "--url=${url}")
endforeach()
if(NOT vcpkg_download_distfile_QUIET)
message(STATUS "Downloading ${vcpkg_download_distfile_URLS} -> ${vcpkg_download_distfile_FILENAME}...")
endif()
set(request_headers)
if(vcpkg_download_distfile_HEADERS)
foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
list(APPEND request_headers "--header=${header}")
endforeach()
endif()
vcpkg_execute_in_download_mode(
COMMAND "$ENV{VCPKG_COMMAND}" x-download "${downloaded_file_path}" "${vcpkg_download_distfile_SHA512}" ${urls} ${request_headers} --debug
OUTPUT_VARIABLE output
ERROR_VARIABLE output
RESULT_VARIABLE failure
WORKING_DIRECTORY "${DOWNLOADS}"
)
if(failure)
message("${output}")
set(download_success 0)
else()
set(download_success 1)
endif()
endif()
if (NOT vcpkg_download_distfile_SILENT_EXIT)
if (NOT download_success)
message(FATAL_ERROR
if(NOT download_success)
message(FATAL_ERROR
" \n"
" Failed to download file.\n"
" If you use a proxy, please check your proxy setting. Possible causes are:\n"
@ -205,23 +241,6 @@ function(vcpkg_download_distfile VAR)
" and (https://github.com/microsoft/vcpkg-tool/pull/77)\n"
" \n"
" Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n")
else()
test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
file(MAKE_DIRECTORY "${downloaded_file_dir}")
file(RENAME ${download_file_path_part} ${downloaded_file_path})
endif()
else()
if (NOT download_success)
message(WARNING
" \n"
" Failed to download file.\n")
else()
test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
file(MAKE_DIRECTORY "${downloaded_file_dir}")
file(RENAME ${download_file_path_part} ${downloaded_file_path})
endif()
endif()
endif()
set(${VAR} ${downloaded_file_path} PARENT_SCOPE)

91
scripts/cmake/vcpkg_from_sourceforge.cmake
View file

@ -67,34 +67,6 @@ Specifies that the default removal of the top level folder should not occur.
#]===]
function(vcpkg_from_sourceforge)
macro(check_file_content)
if (EXISTS ${ARCHIVE})
file(SIZE ${ARCHIVE} DOWNLOAD_FILE_SIZE)
if (DOWNLOAD_FILE_SIZE LESS_EQUAL 1024)
file(READ ${ARCHIVE} _FILE_CONTENT_)
string(FIND "${_FILE_CONTENT_}" "the Sourceforge site is currently in Disaster Recovery mode." OUT_CONTENT)
message("OUT_CONTENT: ${OUT_CONTENT}")
if (OUT_CONTENT EQUAL -1)
set(download_success 1)
else()
file(REMOVE ${ARCHIVE})
endif()
endif()
endif()
endmacro()
macro(check_file_sha512)
file(SHA512 ${ARCHIVE} FILE_HASH)
if(NOT FILE_HASH STREQUAL _vdus_SHA512)
message(FATAL_ERROR
"\nFile does not have expected hash:\n"
" File path: [ ${ARCHIVE} ]\n"
" Expected hash: [ ${_vdus_SHA512} ]\n"
" Actual hash: [ ${FILE_HASH} ]\n"
"${CUSTOM_ERROR_ADVICE}\n")
endif()
endmacro()
set(booleanValueArgs DISABLE_SSL NO_REMOVE_ONE_LEVEL)
set(oneValueArgs OUT_SOURCE_PATH REPO REF SHA512 FILENAME WORKING_DIRECTORY)
set(multipleValuesArgs PATCHES)
@ -154,7 +126,8 @@ function(vcpkg_from_sourceforge)
string(SUBSTRING "${_vdus_SHA512}" 0 10 SANITIZED_REF)
list(APPEND SOURCEFORGE_MIRRORS
set(Z_VCPKG_SOURCEFORGE_MIRRORS ${SOURCEFORGE_MIRRORS})
list(APPEND Z_VCPKG_SOURCEFORGE_MIRRORS
cfhcable # United States
pilotfiber # New York, NY
gigenet # Chicago, IL
@ -176,56 +149,18 @@ function(vcpkg_from_sourceforge)
ufpr # Curitiba, Brazil
tenet # Wynberg, South Africa
)
# Try to use auto-select first
set(DOWNLOAD_URL ${URL}/download)
message(STATUS "Trying auto-select mirror...")
vcpkg_download_distfile(ARCHIVE
URLS "${DOWNLOAD_URL}"
SKIP_SHA512
FILENAME "${_vdus_FILENAME}"
SILENT_EXIT
)
check_file_content()
if (download_success)
check_file_sha512()
else()
message(STATUS "The default mirror is in Disaster Recovery mode, trying other mirrors...")
endif()
if (NOT download_success EQUAL 1)
foreach(SOURCEFORGE_MIRROR ${SOURCEFORGE_MIRRORS})
set(DOWNLOAD_URL ${URL}/download?use_mirror=${SOURCEFORGE_MIRROR})
message(STATUS "Trying mirror ${SOURCEFORGE_MIRROR}...")
vcpkg_download_distfile(ARCHIVE
URLS "${DOWNLOAD_URL}"
SKIP_SHA512
FILENAME "${_vdus_FILENAME}"
SILENT_EXIT
)
if (EXISTS ${ARCHIVE})
set(download_success 1)
check_file_content()
if (download_success)
check_file_sha512()
else()
message(STATUS "Mirror ${SOURCEFORGE_MIRROR} is in Disaster Recovery mode, trying other mirrors...")
endif()
break()
endif()
endforeach()
endif()
if (NOT download_success)
message(FATAL_ERROR [[
Couldn't download source from any of the sourceforge mirrors, please check your network.
If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment
variables to "http[s]://user:password@your-proxy-ip-address:port/".
Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues
]])
endif()
set(URLS "${URL}/download")
foreach(SOURCEFORGE_MIRROR IN LISTS Z_VCPKG_SOURCEFORGE_MIRRORS)
list(APPEND URLS "${URL}/download?use_mirror=${SOURCEFORGE_MIRROR}")
endforeach()
vcpkg_download_distfile(ARCHIVE
URLS ${URLS}
SHA512 "${_vdus_SHA512}"
FILENAME "${_vdus_FILENAME}"
)
vcpkg_extract_source_archive_ex(
OUT_SOURCE_PATH SOURCE_PATH
ARCHIVE "${ARCHIVE}"