extensions-cache-conciliator.sh 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959
  1. #!/bin/sh
  2. # shellcheck disable=SC2039
  3. # shellcheck disable=SC2010
  4. # shellcheck disable=SC2012
  5. set -e
  6. { [ "$EXTENSIONS_CACHE_LOG_LEVEL" != DEBUG ] && [ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ]; } || set -x
  7. run () {
  8. # shellcheck disable=SC2153
  9. [ -n "$EXTENSIONS_REPOSITORY_URLS" ]
  10. [ -n "$1" ]
  11. [ -n "$2" ]
  12. NAMESPACE="$1"
  13. STATEFULSET_NAME="$2"
  14. SERVICEACCOUNT_NAME="${3:-$2}"
  15. PERSISTENTVOLUMECLAIM_NAME="${4:-$2}"
  16. SERVICEACCOUNT_JSON_FILE=serviceaccount.json
  17. STATEFULSET_JSON_FILE=statefulset.json
  18. ANY_IMAGE_REPOSITORY_URL="$(any_image_repository_url && echo true || echo false)"
  19. EXTENSION_METADATA_VERSION=v2
  20. DEFAULT_FLAVOR=pg
  21. DEFAULT_BUILD_ARCH=x86_64
  22. DEFAULT_BUILD_OS=linux
  23. NOT_FOUND_URL_REGEXP='^[^ ]\+ - - \[\([^]]\+\)\] "GET \([^ ]\+\) HTTP\/1\.1" 404 [^ ]\+ "[^"]*" "[^"]*" "[^"]*"$'
  24. while true
  25. do
  26. set +e
  27. (
  28. set -e
  29. echo "Updating indexes..."
  30. pull_indexes
  31. echo "done"
  32. echo
  33. if [ "$ANY_IMAGE_REPOSITORY_URL" = true ]
  34. then
  35. echo "Updating repositories credentials..."
  36. (
  37. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  38. SERVICEACCOUNT="$(kubectl get serviceaccount -n "$NAMESPACE" "$SERVICEACCOUNT_NAME" -o json)"
  39. printf '%s' "$SERVICEACCOUNT" > "$SERVICEACCOUNT_JSON_FILE"
  40. )
  41. update_repositories_credentials
  42. (
  43. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  44. kubectl patch serviceaccount -n "$NAMESPACE" "$SERVICEACCOUNT_NAME" --type merge -p "$(cat "$SERVICEACCOUNT_JSON_FILE")"
  45. )
  46. echo "done"
  47. echo
  48. fi
  49. echo "Updating extensions..."
  50. (
  51. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  52. STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
  53. printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE"
  54. )
  55. (
  56. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  57. get_to_install_extensions > full_to_install_extensions
  58. sort full_to_install_extensions \
  59. | uniq | { grep -v '^$' || true; } > to_install_extensions
  60. TO_INSTALL_EXTENSIONS_JSON_STRING="$(jq -sR . to_install_extensions)"
  61. PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
  62. jq '.metadata.annotations
  63. | if . != null then .pulled_extensions else "" end
  64. | if . != null then . else "" end' "$STATEFULSET_JSON_FILE")"
  65. printf '%s' "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" | jq -r . > already_pulled_to_install_extensions
  66. if [ "$TO_INSTALL_EXTENSIONS_JSON_STRING" != "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" ]
  67. then
  68. while read -r LINE
  69. do
  70. grep -q -xF "$LINE" already_pulled_to_install_extensions || printf '%s\n' "$LINE"
  71. done < to_install_extensions > pull_to_install_extensions
  72. else
  73. true > pull_to_install_extensions
  74. fi
  75. get_not_found_to_install_extensions > not_found_to_install_extensions
  76. )
  77. rm -f pulled_to_install_extensions
  78. touch pulled_to_install_extensions
  79. cat pull_to_install_extensions not_found_to_install_extensions \
  80. | while read -r TO_INSTALL_REPOSITORY PUBLISHER NAME VERSION POSTGRES_VERSION FLAVOR BUILD_ARCH BUILD_OS BUILD
  81. do
  82. try_function pull_extension "$TO_INSTALL_REPOSITORY" "$PUBLISHER" "$NAME" \
  83. "$VERSION" "$POSTGRES_VERSION" "$FLAVOR" "$BUILD_ARCH" "$BUILD_OS" "$BUILD"
  84. if "$RESULT"
  85. then
  86. echo "$TO_INSTALL_REPOSITORY" "$PUBLISHER" "$NAME" \
  87. "$VERSION" "$POSTGRES_VERSION" "$FLAVOR" "$BUILD_ARCH" "$BUILD_OS" "$BUILD" >> pulled_to_install_extensions
  88. else
  89. echo "Warning: error while trying to pull extension " \
  90. "$NAME-$VERSION-$FLAVOR$POSTGRES_VERSION$([ -z "$BUILD" ] || printf '%s' "-build-$BUILD")" \
  91. "(publisher $PUBLISHER from $TO_INSTALL_REPOSITORY for $BUILD_ARCH/$BUILD_OS)" >&2
  92. fi
  93. done
  94. (
  95. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  96. ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
  97. jq '.metadata.annotations.pulled_extensions' "$STATEFULSET_JSON_FILE")"
  98. PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
  99. cat pulled_to_install_extensions already_pulled_to_install_extensions \
  100. | sort | uniq | jq -sR .)"
  101. if [ "$ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" != "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" ]
  102. then
  103. RUNNING_IMAGES="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
  104. --template '{{ range .spec.template.spec.containers }}{{ printf "%s\n" .image }}{{ end }}')"
  105. RUNNING_IMAGES="$(printf '%s' "$RUNNING_IMAGES" | sort)"
  106. REQUIRED_IMAGES="$(jq '.spec.template.spec.containers[].image' "$STATEFULSET_JSON_FILE")"
  107. REQUIRED_IMAGES="$(printf '%s' "$REQUIRED_IMAGES" | sort)"
  108. if [ "$RUNNING_IMAGES" != "$REQUIRED_IMAGES" ]
  109. then
  110. touch /tmp/need-restart
  111. fi
  112. STATEFULSET="$(cat "$STATEFULSET_JSON_FILE")"
  113. printf '%s' "$STATEFULSET" \
  114. | jq ".metadata.annotations.pulled_extensions = $PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" \
  115. > "$STATEFULSET_JSON_FILE"
  116. cp "$STATEFULSET_JSON_FILE" "$STATEFULSET_JSON_FILE.new"
  117. until kubectl replace statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
  118. --cascade orphan -f "$STATEFULSET_JSON_FILE.new"
  119. do
  120. sleep 5
  121. STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
  122. printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE.lastest"
  123. jq -s '.[0].spec.template = .[1].spec.template | .[0]' \
  124. "$STATEFULSET_JSON_FILE.lastest" "$STATEFULSET_JSON_FILE" > "$STATEFULSET_JSON_FILE.new"
  125. done
  126. fi
  127. if ! any_image_repository_url && test -f /tmp/need-restart
  128. then
  129. kubectl delete pod -n "$NAMESPACE" "$STATEFULSET_NAME-0"
  130. fi
  131. )
  132. echo "done"
  133. echo
  134. )
  135. EXIT_CODE="$?"
  136. if [ "$EXIT_CODE" = 0 ]
  137. then
  138. if ! test -f /tmp/need-restart
  139. then
  140. touch /tmp/extensions-cache-ready
  141. fi
  142. echo
  143. echo "...wait for next reconciliation cycle"
  144. echo
  145. sleep 10
  146. else
  147. echo
  148. echo "...an error occurred during reconciliation cycle, retrying in 10 seconds"
  149. echo
  150. sleep 10
  151. fi
  152. done
  153. }
  154. any_image_repository_url() {
  155. local EXTENSIONS_REPOSITORY_URL
  156. for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
  157. do
  158. if is_image_repository_url "$EXTENSIONS_REPOSITORY_URL"
  159. then
  160. return
  161. fi
  162. done
  163. return 1
  164. }
  165. is_image_repository_url() {
  166. printf '%s' "$1" | grep -q "[?&]imageTemplate="
  167. }
  168. is_image_repository_with_regcred_secret_url() {
  169. printf '%s' "$1" | grep "[?&]imageTemplate=" | grep -q "[?&]imageRegcredSecret="
  170. }
  171. is_proxied_repository_url() {
  172. printf '%s' "$1" | grep -q "[?&]proxyUrl="
  173. }
  174. is_cache_timeout_repository_url() {
  175. printf '%s' "$1" | grep -q "[?&]cacheTimeout="
  176. }
  177. is_skip_hostname_verification_repository_url() {
  178. printf '%s' "$1" | grep -q "[?&]skipHostnameVerification="
  179. }
  180. is_set_http_scheme_url() {
  181. printf '%s' "$1" | grep -q "[?&]setHttpScheme="
  182. }
  183. get_image_template_from_url() {
  184. local IMAGE_TEMPLATE
  185. IMAGE_TEMPLATE="$(printf '%s' "$1" | grep "[?&]imageTemplate=")"
  186. IMAGE_TEMPLATE="$(printf '%s' "$IMAGE_TEMPLATE" \
  187. | sed 's/^.*[?&]imageTemplate=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  188. IMAGE_TEMPLATE="$(printf '%s' "$IMAGE_TEMPLATE" | urldecode)"
  189. # shellcheck disable=SC2016
  190. if printf '%s' "$IMAGE_TEMPLATE" | grep -q '\([`;"]\|\$(\)'
  191. then
  192. # shellcheck disable=SC2016
  193. echo 'Forbidden strings [`|;|"|$(] found in parameter imageTemplate of URL '"$1" >&2
  194. return 1
  195. fi
  196. printf '%s' "$IMAGE_TEMPLATE"
  197. }
  198. get_image_regcred_secret_from_url() {
  199. local REPOSITORY_CREDENTIAL_SECRET
  200. REPOSITORY_CREDENTIAL_SECRET="$(printf '%s' "$1" | grep "[?&]imageRegcredSecret=")"
  201. REPOSITORY_CREDENTIAL_SECRET="$(printf '%s' "$REPOSITORY_CREDENTIAL_SECRET" \
  202. | sed 's/^.*[?&]imageRegcredSecret=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  203. REPOSITORY_CREDENTIAL_SECRET="$(printf '%s' "$REPOSITORY_CREDENTIAL_SECRET" | urldecode)"
  204. printf '%s' "$REPOSITORY_CREDENTIAL_SECRET"
  205. }
  206. get_proxy_from_url() {
  207. local PROXY_URL
  208. PROXY_URL="$(printf '%s' "$1" | grep "[?&]proxyUrl=")"
  209. PROXY_URL="$(printf '%s' "$PROXY_URL" \
  210. | sed 's/^.*[?&]proxyUrl=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  211. PROXY_URL="$(printf '%s' "$PROXY_URL" | urldecode)"
  212. printf '%s' "$PROXY_URL"
  213. }
  214. get_cache_timeout_from_url() {
  215. local CACHE_TIMEOUT
  216. CACHE_TIMEOUT="$(printf '%s' "$1" | grep "[?&]cacheTimeout=")"
  217. CACHE_TIMEOUT="$(printf '%s' "$CACHE_TIMEOUT" \
  218. | sed 's/^.*[?&]cacheTimeout=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  219. CACHE_TIMEOUT="$(printf '%s' "$CACHE_TIMEOUT" | urldecode)"
  220. printf '%s' "$CACHE_TIMEOUT"
  221. }
  222. get_skip_hostname_verification_from_url() {
  223. local SKIP_HOSTNAME_VERIFICATION
  224. SKIP_HOSTNAME_VERIFICATION="$(printf '%s' "$1" | grep "[?&]skipHostnameVerification=")"
  225. SKIP_HOSTNAME_VERIFICATION="$(printf '%s' "$SKIP_HOSTNAME_VERIFICATION" \
  226. | sed 's/^.*[?&]skipHostnameVerification=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  227. SKIP_HOSTNAME_VERIFICATION="$(printf '%s' "$SKIP_HOSTNAME_VERIFICATION" | urldecode)"
  228. printf '%s' "$SKIP_HOSTNAME_VERIFICATION"
  229. }
  230. get_set_http_scheme_from_url() {
  231. local SET_HTTP_SCHEME
  232. SET_HTTP_SCHEME="$(printf '%s' "$1" | grep "[?&]setHttpScheme=")"
  233. SET_HTTP_SCHEME="$(printf '%s' "$SET_HTTP_SCHEME" \
  234. | sed 's/^.*[?&]setHttpScheme=\([^?&]\+\)\([?&].*\)\?$/\1/')"
  235. SET_HTTP_SCHEME="$(printf '%s' "$SET_HTTP_SCHEME" | urldecode)"
  236. printf '%s' "$SET_HTTP_SCHEME"
  237. }
  238. pull_indexes() {
  239. local INDEX_NAME EXTENSIONS_REPOSITORY_URL
  240. local INDEX
  241. for INDEX_NAME in index hashes
  242. do
  243. if ! test -s "unwrapped-${INDEX_NAME}.json"
  244. then
  245. echo '{}' > "unwrapped-${INDEX_NAME}.json"
  246. fi
  247. INDEX=-1
  248. for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
  249. do
  250. INDEX="$((INDEX + 1))"
  251. if ! is_index_cache_expired "$EXTENSIONS_REPOSITORY_URL" "$INDEX_NAME" "$INDEX"
  252. then
  253. continue
  254. fi
  255. local CURL_EXTRA_OPTS=""
  256. if is_proxied_repository_url "$EXTENSIONS_REPOSITORY_URL"
  257. then
  258. local PROXY_URL
  259. PROXY_URL="$(get_proxy_from_url "$EXTENSIONS_REPOSITORY_URL")"
  260. CURL_EXTRA_OPTS="$CURL_EXTRA_OPTS --proxy $PROXY_URL"
  261. fi
  262. if is_skip_hostname_verification_repository_url "$EXTENSIONS_REPOSITORY_URL"
  263. then
  264. local SKIP_HOSTNAME_VERIFICATION
  265. SKIP_HOSTNAME_VERIFICATION="$(get_skip_hostname_verification_from_url "$EXTENSIONS_REPOSITORY_URL")"
  266. if [ "$SKIP_HOSTNAME_VERIFICATION" = true ]
  267. then
  268. CURL_EXTRA_OPTS="$CURL_EXTRA_OPTS -k"
  269. fi
  270. fi
  271. curl -f -s -L $CURL_EXTRA_OPTS "${EXTENSIONS_REPOSITORY_URL%%\?*}/${EXTENSION_METADATA_VERSION}/${INDEX_NAME}.json" > "last-${INDEX_NAME}.json"
  272. jq -s '.[0] as $last | .[1] as $current_unwrapped
  273. | $last
  274. | .publishers = (.publishers | map(
  275. {
  276. key: .id,
  277. value: .
  278. }
  279. ) | from_entries)
  280. | .extensions = (.extensions | map(
  281. {
  282. key: .name,
  283. value: (.repository = "'"${EXTENSIONS_REPOSITORY_URL%%\?*}"'")
  284. }
  285. ) | from_entries)
  286. | . as $last_unwrapped
  287. | $current_unwrapped * $last_unwrapped
  288. ' "last-${INDEX_NAME}.json" "unwrapped-${INDEX_NAME}.json" > "new-unwrapped-${INDEX_NAME}.json"
  289. local EXTENSIONS_REPOSITORY_PATH="${EXTENSIONS_REPOSITORY_URL#*://}"
  290. EXTENSIONS_REPOSITORY_PATH="${EXTENSIONS_REPOSITORY_PATH%%\?*}"
  291. mkdir -p "$EXTENSIONS_REPOSITORY_PATH/$EXTENSION_METADATA_VERSION/"
  292. mv "last-${INDEX_NAME}.json" "$EXTENSIONS_REPOSITORY_PATH/$EXTENSION_METADATA_VERSION/${INDEX_NAME}.json"
  293. mv "new-unwrapped-${INDEX_NAME}.json" "unwrapped-${INDEX_NAME}.json"
  294. date +%s > "${INDEX_NAME}-${INDEX}.timestamp"
  295. done
  296. jq '.extensions = (.extensions | to_entries | map(.value))
  297. | .publishers = (.publishers | to_entries | map(.value))
  298. ' "unwrapped-${INDEX_NAME}.json" > "merged-${INDEX_NAME}.json"
  299. mv "merged-${INDEX_NAME}.json" "${INDEX_NAME}.json"
  300. done
  301. }
  302. is_any_index_expired() {
  303. local INDEX_NAME
  304. for INDEX_NAME in index hashes
  305. do
  306. local INDEX=-1
  307. for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
  308. do
  309. INDEX="$((INDEX + 1))"
  310. if is_index_cache_expired "$EXTENSIONS_REPOSITORY_URL" "$INDEX_NAME" "$INDEX"
  311. then
  312. return
  313. fi
  314. done
  315. done
  316. return 1
  317. }
  318. is_index_cache_expired() {
  319. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] \
  320. && true || false
  321. local EXTENSIONS_REPOSITORY_URL="$1"
  322. local INDEX_NAME="$2"
  323. local INDEX="$3"
  324. local CACHE_TIMEOUT=3600
  325. if is_cache_timeout_repository_url "$EXTENSIONS_REPOSITORY_URL"
  326. then
  327. CACHE_TIMEOUT="$(get_cache_timeout_from_url "$EXTENSIONS_REPOSITORY_URL")"
  328. fi
  329. ! test -f "${INDEX_NAME}-${INDEX}.timestamp" \
  330. || [ "$(date +%s)" -ge "$(( $(cat "${INDEX_NAME}-${INDEX}.timestamp") + CACHE_TIMEOUT ))" ]
  331. }
  332. update_repositories_credentials() {
  333. local EXTENSIONS_REPOSITORY_URL SERVICEACCOUNT
  334. for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
  335. do
  336. if is_image_repository_with_regcred_secret_url "$EXTENSIONS_REPOSITORY_URL"
  337. then
  338. REPOSITORY_CREDENTIAL_SECRET="$(get_image_regcred_secret_from_url "$EXTENSIONS_REPOSITORY_URL")"
  339. if jq '([] + .imagePullSecrets)|any(.name == "'"$REPOSITORY_CREDENTIAL_SECRET"'")' "$SERVICEACCOUNT_JSON_FILE" | grep -q false
  340. then
  341. (
  342. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  343. SERVICEACCOUNT="$(jq ".imagePullSecrets = .imagePullSecrets + [{name: \"$REPOSITORY_CREDENTIAL_SECRET\"}]" "$SERVICEACCOUNT_JSON_FILE")"
  344. printf '%s' "$SERVICEACCOUNT" > "$SERVICEACCOUNT_JSON_FILE"
  345. )
  346. fi
  347. fi
  348. done
  349. }
  350. get_to_install_extensions() {
  351. (
  352. CLUSTER_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
  353. && printf %s "$ALLOWED_NAMESPACES" \
  354. | tr ' ' '\n' \
  355. | xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
  356. "kubectl get sgcluster -n @NAMESPACE -o json | jq '.items[]'" \
  357. || kubectl get sgcluster -A -o json | jq '.items[]')"
  358. DISTRIBUTEDLOGS_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
  359. && printf %s "$ALLOWED_NAMESPACES" \
  360. | tr ' ' '\n' \
  361. | xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
  362. "kubectl get sgdistributedlogs -n @NAMESPACE -o json | jq '.items[]'" \
  363. || kubectl get sgdistributedlogs -A -o json | jq '.items[]')"
  364. printf '%s' "$CLUSTER_EXTENSIONS"
  365. printf '%s' "$DISTRIBUTEDLOGS_EXTENSIONS"
  366. ) | jq -r -s '.[]
  367. | select(has("status") and (.status | has("arch") and has("os")))
  368. | .status as $status
  369. | .spec.toInstallPostgresExtensions
  370. | if . != null then . else [] end
  371. | .[]
  372. | .repository + " "
  373. + .publisher + " "
  374. + .name + " "
  375. + .version + " "
  376. + .postgresVersion
  377. + " " + (.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
  378. + " " + $status.arch
  379. + " " + $status.os
  380. + " " + (.build | if . != null then . else "" end)
  381. '
  382. jq -r '
  383. .extensions[] | . as $extension
  384. | .versions[] | . as $version
  385. | .availableFor = (.availableFor
  386. | sort_by(if .build == null then 0 else (.build | split(".")
  387. | (.[0] | tonumber | . * 10000) + (.[1] | split("-")[0] | tonumber)) end)
  388. | reduce .[] as $availableFor ({};
  389. . as $result | ($availableFor.postgresVersion | if . != null then . else "any" end)
  390. + "-" + ($availableFor.arch | if . != null then . else "'"$DEFAULT_BUILD_ARCH"'" end)
  391. + "-" + ($availableFor.os | if . != null then . else "'"$DEFAULT_BUILD_OS"'" end) | . as $key
  392. | $result | .[$key] = $availableFor) | to_entries | map(.value))
  393. | .availableFor[] | . as $availableFor
  394. | select('"$EXTENSIONS_CACHE_PRELOADED_EXTENSIONS"' | any(. as $test
  395. | $extension.publisher
  396. + "/" + ($availableFor.arch | if . != null then . else "'"$DEFAULT_BUILD_ARCH"'" end)
  397. + "/" + ($availableFor.os | if . != null then . else "'"$DEFAULT_BUILD_OS"'" end)
  398. + "/" + $extension.name + "-" + $version.version + "-"
  399. + ($availableFor.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
  400. + $availableFor.postgresVersion
  401. + ($availableFor.build | if . != null then "-build-" + . else "" end)
  402. | test($test; "")))
  403. | $extension.repository + " "
  404. + $extension.publisher + " "
  405. + $extension.name + " "
  406. + $version.version + " "
  407. + $availableFor.postgresVersion
  408. + " " + ($availableFor.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
  409. + " " + ($availableFor.arch | if . != null then . else "'"$DEFAULT_BUILD_ARCH"'" end)
  410. + " " + ($availableFor.os | if . != null then . else "'"$DEFAULT_BUILD_OS"'" end)
  411. + " " + ($availableFor.build | if . != null then . else "" end)' \
  412. index.json || echo "Error while applying filter $EXTENSIONS_CACHE_PRELOADED_EXTENSIONS to preload extensions" >&2
  413. }
  414. get_not_found_to_install_extensions() {
  415. rm -f last_date_time_found
  416. grep "$NOT_FOUND_URL_REGEXP" /var/log/nginx/access.log \
  417. | sed -u "s/$NOT_FOUND_URL_REGEXP/\1 \2/" \
  418. | while read -r DATE_TIME TIME_ZONE NOT_FOUND_URL
  419. do
  420. if test -f last_date_time \
  421. && [ "$DATE_TIME $TIME_ZONE" = "$(cat last_date_time)" ] \
  422. && ! test -f last_date_time_found
  423. then
  424. touch last_date_time_found
  425. continue
  426. fi
  427. try_function get_extension_from_url "$NOT_FOUND_URL"
  428. if "$RESULT"
  429. then
  430. printf '%s %s' "$DATE_TIME" "$TIME_ZONE" > last_date_time
  431. else
  432. echo "Warning: error while trying to extract extension from $NOT_FOUND_URL" >&2
  433. fi
  434. done
  435. }
  436. get_extension_from_url() {
  437. [ -n "$1" ] && true || false
  438. local NOT_FOUND_URL="$1"
  439. local EXTENSIONS_REPOSITORY_URL NOT_FOUND_URL_PATH EXTENSIONS_REPOSITORY_URL_PATH
  440. local REPOSITORY PUBLISHER BUILD_ARCH BUILD_OS EXTENSION_PACKAGE EXTENSION_ID_JSON
  441. EXTENSIONS_REPOSITORY_URL="$(find_repository_base_url "$NOT_FOUND_URL")"
  442. if is_image_repository_url "$EXTENSIONS_REPOSITORY_URL"
  443. then
  444. return
  445. fi
  446. NOT_FOUND_URL_PATH="${NOT_FOUND_URL#*://}"
  447. NOT_FOUND_URL_PATH="${NOT_FOUND_URL_PATH%%\?*}"
  448. EXTENSIONS_REPOSITORY_URL_PATH="${EXTENSIONS_REPOSITORY_URL%%\?*}"
  449. EXTENSIONS_REPOSITORY_URL_PATH="${EXTENSIONS_REPOSITORY_URL_PATH#*://}"
  450. NOT_FOUND_URL_RESOURCE_PATH="${NOT_FOUND_URL_PATH#${EXTENSIONS_REPOSITORY_URL_PATH}}"
  451. NOT_FOUND_URL_RESOURCE_PATH="${NOT_FOUND_URL_RESOURCE_PATH#/}"
  452. if [ "$(printf '%s' "$NOT_FOUND_URL_RESOURCE_PATH" | fold -w 1 | grep -c /)" -eq 3 ]
  453. then
  454. REPOSITORY="${NOT_FOUND_URL%%$NOT_FOUND_URL_RESOURCE_PATH\?*}"
  455. PUBLISHER="$(printf '%s' "$NOT_FOUND_URL_RESOURCE_PATH" | cut -d '/' -f 1)"
  456. BUILD_ARCH="$(printf '%s' "$NOT_FOUND_URL_RESOURCE_PATH" | cut -d '/' -f 2)"
  457. BUILD_OS="$(printf '%s' "$NOT_FOUND_URL_RESOURCE_PATH" | cut -d '/' -f 3)"
  458. EXTENSION_PACKAGE="$(printf '%s' "$NOT_FOUND_URL_RESOURCE_PATH" | cut -d '/' -f 4)"
  459. EXTENSION_PACKAGE="${EXTENSION_PACKAGE%.tar}"
  460. EXTENSION_ID_JSON="$(printf '%s' "$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE" | jq -sR .)"
  461. jq -r '
  462. .extensions[] | . as $extension
  463. | .versions[] | . as $version
  464. | .availableFor | map(. as $availableFor
  465. | select('"$EXTENSION_ID_JSON"' ==
  466. $extension.publisher + "/"
  467. + ($availableFor.arch | if . != null then . else "'"$DEFAULT_BUILD_ARCH"'" end) + "/"
  468. + ($availableFor.os | if . != null then . else "'"$DEFAULT_BUILD_OS"'" end) + "/"
  469. + $extension.name + "-" + $version.version + "-"
  470. + ($availableFor.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
  471. + $availableFor.postgresVersion
  472. + ($availableFor.build | if . != null then "-build-" + . else "" end))
  473. | $extension.repository + " "
  474. + $extension.publisher + " "
  475. + $extension.name + " "
  476. + $version.version + " "
  477. + $availableFor.postgresVersion
  478. + " " + ($availableFor.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
  479. + " " + ($availableFor.arch | if . != null then . else "'"$DEFAULT_BUILD_ARCH"'" end)
  480. + " " + ($availableFor.os | if . != null then . else "'"$DEFAULT_BUILD_OS"'" end)
  481. + " " + ($availableFor.build | if . != null then . else "" end))
  482. | .[]' \
  483. index.json
  484. fi
  485. }
  486. pull_extension() {
  487. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] && [ -n "$4" ] \
  488. && [ -n "$5" ] && [ -n "$6" ] && [ -n "$7" ] && [ -n "$8" ] \
  489. && true || false
  490. local TO_INSTALL_REPOSITORY="$1"
  491. local PUBLISHER="$2"
  492. local NAME="$3"
  493. local VERSION="$4"
  494. local POSTGRES_VERSION="$5"
  495. local FLAVOR="$6"
  496. local BUILD_ARCH="$7"
  497. local BUILD_OS="$8"
  498. local BUILD="$9"
  499. local REPOSITORY="$TO_INSTALL_REPOSITORY"
  500. local EXTENSION_PACKAGE
  501. EXTENSION_PACKAGE="$NAME-$VERSION-$FLAVOR$POSTGRES_VERSION$([ -z "$BUILD" ] || printf '%s' "-build-$BUILD")"
  502. echo " * Required extension $EXTENSION_PACKAGE"
  503. EXTENSIONS_REPOSITORY_URL="$(find_repository_base_url "$REPOSITORY")"
  504. if [ "$ANY_IMAGE_REPOSITORY_URL" != true ] \
  505. || ! is_image_repository_url "$EXTENSIONS_REPOSITORY_URL"
  506. then
  507. download_extension "$EXTENSIONS_REPOSITORY_URL" "$REPOSITORY" "$PUBLISHER" "$BUILD_ARCH" "$BUILD_OS"\
  508. "$EXTENSION_PACKAGE"
  509. else
  510. add_extension_image_to_statefulset "$EXTENSIONS_REPOSITORY_URL" "$REPOSITORY" "$PUBLISHER" "$NAME" \
  511. "$VERSION" "$POSTGRES_VERSION" "$FLAVOR" "$BUILD_ARCH" "$BUILD_OS" "$BUILD"
  512. fi
  513. }
  514. find_repository_base_url() {
  515. [ -n "$1" ] && true || false
  516. local REPOSITORY="$1"
  517. local IS_PROXY_URL_AND_SET_HTTP_SCHEME=false
  518. if is_proxied_repository_url "$REPOSITORY"
  519. then
  520. local PROXY_URL
  521. PROXY_URL="$(get_proxy_from_url "$REPOSITORY")"
  522. if is_set_http_scheme_url "$PROXY_URL"
  523. then
  524. IS_PROXY_URL_AND_SET_HTTP_SCHEME="$(get_set_http_scheme_from_url "$PROXY_URL")"
  525. fi
  526. fi
  527. ! test -f "${INDEX_NAME}-${INDEX}.timestamp" \
  528. || [ "$(date +%s)" -ge "$(( $(cat "${INDEX_NAME}-${INDEX}.timestamp") + CACHE_TIMEOUT ))" ]
  529. local EXTENSIONS_REPOSITORY_URL
  530. for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
  531. do
  532. local TEST_EXTENSIONS_REPOSITORY_URL="$EXTENSIONS_REPOSITORY_URL"
  533. if [ "$IS_PROXY_URL_AND_SET_HTTP_SCHEME" = true ]
  534. then
  535. TEST_EXTENSIONS_REPOSITORY_URL="http://${EXTENSIONS_REPOSITORY_URL#*://}"
  536. fi
  537. if printf '%s' "$TEST_EXTENSIONS_REPOSITORY_URL" | grep -q "^${REPOSITORY%%\?*}" \
  538. || printf '%s' "$REPOSITORY" | grep -q "^${TEST_EXTENSIONS_REPOSITORY_URL%%\?*}"
  539. then
  540. printf '%s' "$EXTENSIONS_REPOSITORY_URL"
  541. return
  542. fi
  543. done
  544. echo "Warning: Could not found repository $REPOSITORY between following repositories:" >&2
  545. echo >&2
  546. echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n' >&2
  547. echo >&2
  548. return 1
  549. }
  550. download_extension() {
  551. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] && [ -n "$4" ] \
  552. && [ -n "$5" ] && [ -n "$6" ] && true || false
  553. local EXTENSIONS_REPOSITORY_URL="$1"
  554. local REPOSITORY="$2"
  555. local PUBLISHER="$3"
  556. local BUILD_ARCH="$4"
  557. local BUILD_OS="$5"
  558. local EXTENSION_PACKAGE="$6"
  559. local TEMP="$(shuf -i 0-65535 -n 1)"
  560. local REPOSITORY_PATH="${REPOSITORY#*://}"
  561. if ! test -f "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar"
  562. then
  563. echo " + Downloading from $REPOSITORY/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar"
  564. mkdir -p "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS"
  565. local CURL_EXTRA_OPTS=""
  566. if is_proxied_repository_url "$EXTENSIONS_REPOSITORY_URL"
  567. then
  568. local PROXY_URL
  569. PROXY_URL="$(get_proxy_from_url "$EXTENSIONS_REPOSITORY_URL")"
  570. CURL_EXTRA_OPTS="--proxy $PROXY_URL"
  571. fi
  572. if is_skip_hostname_verification_repository_url "$EXTENSIONS_REPOSITORY_URL"
  573. then
  574. local SKIP_HOSTNAME_VERIFICATION
  575. SKIP_HOSTNAME_VERIFICATION="$(get_skip_hostname_verification_from_url "$EXTENSIONS_REPOSITORY_URL")"
  576. if [ "$SKIP_HOSTNAME_VERIFICATION" = true ]
  577. then
  578. CURL_EXTRA_OPTS="$CURL_EXTRA_OPTS -k"
  579. fi
  580. fi
  581. curl -f -s -L -I $CURL_EXTRA_OPTS "$REPOSITORY/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar" \
  582. > "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.headers.$TEMP"
  583. CONTENT_LENGTH="$(grep -i 'Content-Length' "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.headers.$TEMP")"
  584. CONTENT_LENGTH="$(printf '%s' "$CONTENT_LENGTH" | tr -d '[:space:]' | cut -d ':' -f 2)"
  585. if [ "$CONTENT_LENGTH" -ge 0 ]
  586. then
  587. apply_extension_cache_retention_policy "$CONTENT_LENGTH"
  588. else
  589. echo "Warning: Can not determine Content-Length for URL $REPOSITORY/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar"
  590. fi
  591. curl -f -s -L $CURL_EXTRA_OPTS "$REPOSITORY/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar" \
  592. -o "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar.tmp.$TEMP"
  593. mv "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar.tmp.$TEMP" \
  594. "$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar"
  595. else
  596. echo " . Already downloaded from $REPOSITORY/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_PACKAGE.tar"
  597. fi
  598. }
  599. apply_extension_cache_retention_policy() {
  600. [ "$1" -ge 0 ] && true || false
  601. local REQUIRED_BYTES="$1"
  602. local AVAILABLE_BYTES BLOCK_SIZE REQUIRED_BYTES_WITH_BLOCK_SIZE
  603. AVAILABLE_BYTES="$(df -B 1 . | tail -n 1 | tr -s '[:space:]' ':' | cut -d : -f 4)"
  604. BLOCK_SIZE="$(stat -fc '%s' .)"
  605. REQUIRED_BYTES_WITH_BLOCK_SIZE="$((REQUIRED_BYTES + BLOCK_SIZE))"
  606. if [ "$AVAILABLE_BYTES" -lt "$REQUIRED_BYTES_WITH_BLOCK_SIZE" ]
  607. then
  608. echo "Required $REQUIRED_BYTES_WITH_BLOCK_SIZE bytes but available $AVAILABLE_BYTES, removing files with oldest last access..."
  609. while true
  610. do
  611. if ! ls -R -- * 2>&1 | grep ':$' | sed 's/:$//' \
  612. | while read -r DIR
  613. do
  614. ls -1 "$DIR"
  615. done \
  616. | grep -q .
  617. then
  618. echo "Warning: Can not free more space on device!"
  619. break
  620. fi
  621. ls -R -- * 2>&1 | grep ':$' | sed 's/:$//' \
  622. | while read -r DIR
  623. do
  624. ls -1 "$DIR" \
  625. | while read -r FILE
  626. do
  627. ! test -f "$DIR/$FILE" || stat --format '%X:%n' "$DIR/$FILE"
  628. done
  629. done \
  630. | sort -t : -k 1n | grep -v '\.json$' | head -n 1 | cut -d : -f 2 \
  631. | while read -r FILE
  632. do
  633. echo "Removing $FILE of $(stat '%s' "$FILE")"
  634. rm -f "$FILE"
  635. done
  636. AVAILABLE_BYTES="$(df -B 1 . | tail -n 1 | tr '[:space:]' ':' | cut -d : -f 4)"
  637. if [ "$AVAILABLE_BYTES" -lt "$REQUIRED_BYTES_WITH_BLOCK_SIZE" ]
  638. then
  639. break
  640. fi
  641. done
  642. echo "done"
  643. echo
  644. fi
  645. }
  646. add_extension_image_to_statefulset() {
  647. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] && [ -n "$4" ] \
  648. && [ -n "$5" ] && [ -n "$6" ] && [ -n "$7" ] && [ -n "$8" ] \
  649. && [ -n "$9" ] && true || false
  650. local EXTENSIONS_REPOSITORY_URL="$1"
  651. local REPOSITORY="$2"
  652. local PUBLISHER="$3"
  653. local NAME="$4"
  654. local VERSION="$5"
  655. local POSTGRES_VERSION="$6"
  656. local FLAVOR="$7"
  657. local BUILD_ARCH="$8"
  658. local BUILD_OS="$9"
  659. local BUILD="$10"
  660. local IMAGE_TEMPLATE COMPONENT COMPONENT_VERSION HASH IMAGE_NAME CONTAINER STATEFULSET
  661. IMAGE_TEMPLATE="$(get_image_template_from_url "$EXTENSIONS_REPOSITORY_URL")"
  662. COMPONENT="$(jq -r ".extensions[\"$NAME\"]
  663. | .component" unwrapped-hashes.json)"
  664. # shellcheck disable=SC2034
  665. COMPONENT_VERSION="$(jq -r ".extensions[\"$NAME\"]
  666. | .versions[] | select(.version == \"$VERSION\")
  667. | .componentVersion | if . != null then . else \"\" end" unwrapped-hashes.json)"
  668. if [ -z "$COMPONENT_VERSION" ]
  669. then
  670. # TODO: this is a mock that have to be fixed
  671. # in the extensions repository by adding a
  672. # .extensions[].versions[].componentVersion field
  673. # in the hashes.json index
  674. COMPONENT_VERSION="$VERSION"
  675. case "$COMPONENT" in
  676. (core|contrib)
  677. COMPONENT_VERSION="$COMPONENT"
  678. ;;
  679. (citus)
  680. COMPONENT_VERSION="10.1.2"
  681. ;;
  682. (healpix)
  683. # shellcheck disable=SC2034
  684. COMPONENT_VERSION="1.0"
  685. ;;
  686. esac
  687. fi
  688. if [ -z "$COMPONENT_VERSION" ]
  689. then
  690. echo " ! Can not retrieve component version for extension $REPOSITORY $PUBLISHER $NAME $VERSION $FLAVOR$POSTGRES_VERSION $BUILD $BUILD_ARCH $BUILD_OS"
  691. return 1
  692. fi
  693. # shellcheck disable=SC2034
  694. HASH="$(jq -r ".extensions[\"$NAME\"]
  695. | .versions[] | select(.version == \"$VERSION\")
  696. | .availableFor[] | select(.postgresVersion == \"$POSTGRES_VERSION\"
  697. and (.flavor == \"$FLAVOR\" or (.flavor == null and \"$FLAVOR\" == \"$DEFAULT_FLAVOR\"))
  698. and (.build == \"$BUILD\" or (.build == null and \"$BUILD\" == \"\")))
  699. | .buildHash" unwrapped-hashes.json)"
  700. if [ -z "$HASH" ]
  701. then
  702. echo " ! Can not retrieve hash for extension $REPOSITORY $PUBLISHER $NAME $VERSION $FLAVOR$POSTGRES_VERSION $BUILD $BUILD_ARCH $BUILD_OS"
  703. return 1
  704. fi
  705. # shellcheck disable=SC2034
  706. POSTGRES_EXACT_VERSION="$(jq -r ".extensions[\"plpgsql\"]
  707. | .versions[]
  708. | .availableFor[] | select(.postgresVersion == \"$POSTGRES_VERSION\"
  709. and (.flavor == \"$FLAVOR\" or (.flavor == null and \"$FLAVOR\" == \"$DEFAULT_FLAVOR\"))
  710. and (.build == \"$BUILD\" or (.build == null and \"$BUILD\" == \"\")))
  711. | .postgresVersion" unwrapped-hashes.json)"
  712. if [ -z "$POSTGRES_EXACT_VERSION" ]
  713. then
  714. # TODO: this is a mock that have to be fixed
  715. # in the extensions repository by adding a
  716. # .extensions[].versions[].availableFor[].postgresExactVersion field
  717. # in the hashes.json index
  718. POSTGRES_EXACT_VERSION="$POSTGRES_VERSION"
  719. case "$POSTGRES_VERSION" in
  720. (11)
  721. POSTGRES_EXACT_VERSION="11.13"
  722. ;;
  723. (12)
  724. POSTGRES_EXACT_VERSION="12.8"
  725. ;;
  726. (13)
  727. # shellcheck disable=SC2034
  728. POSTGRES_EXACT_VERSION="13.4"
  729. ;;
  730. esac
  731. fi
  732. if [ -z "$POSTGRES_EXACT_VERSION" ]
  733. then
  734. echo " ! Can not retrieve postgres exact version for extension $REPOSITORY $PUBLISHER $NAME $VERSION $FLAVOR$POSTGRES_VERSION $BUILD $BUILD_ARCH $BUILD_OS"
  735. return 1
  736. fi
  737. IMAGE_NAME="$(eval "echo \"$IMAGE_TEMPLATE\"")"
  738. if jq '.spec.template.spec.containers|any(.image == "'"$IMAGE_NAME"'")' "$STATEFULSET_JSON_FILE" | grep -q false
  739. then
  740. echo " + Add extension image $IMAGE_NAME"
  741. CONTAINER="$(get_extension_container_as_yaml "$IMAGE_NAME" "$REPOSITORY" "$PUBLISHER" "$BUILD_ARCH" "$BUILD_OS")"
  742. (
  743. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  744. STATEFULSET="$(jq ".spec.template.spec.containers = .spec.template.spec.containers + [$CONTAINER]" "$STATEFULSET_JSON_FILE")"
  745. printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE"
  746. )
  747. else
  748. echo " . Already added image $IMAGE_NAME"
  749. fi
  750. }
  751. get_extension_container_as_yaml() {
  752. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] && [ -n "$4" ] \
  753. && [ -n "$5" ] && true || false
  754. local IMAGE_NAME="$1"
  755. local REPOSITORY="$2"
  756. local PUBLISHER="$3"
  757. local BUILD_ARCH="$4"
  758. local BUILD_OS="$5"
  759. local REPOSITORY_PATH="${REPOSITORY#*://}"
  760. cat << EOF
  761. {
  762. name: "extension-$(jq -r '(.spec.template.spec.containers | length) - 2' "$STATEFULSET_JSON_FILE")",
  763. image: "$IMAGE_NAME",
  764. env: [{
  765. name: "EXTENSIONS_CACHE_LOG_LEVEL",
  766. value: "$EXTENSIONS_CACHE_LOG_LEVEL"
  767. }],
  768. command: [ "sh", "-ec", $(cat << INNER_EOF | jq -sR .
  769. cd /opt/app-root/src
  770. sh /usr/local/bin/extensions-cache-conciliator.sh provide '$REPOSITORY_PATH' '$PUBLISHER' '$BUILD_ARCH' '$BUILD_OS'
  771. INNER_EOF
  772. ) ],
  773. volumeMounts: [{
  774. name: "extensions-cache-config",
  775. mountPath: "/usr/local/bin/extensions-cache-conciliator.sh",
  776. subPath: "extensions-cache-conciliator.sh",
  777. readOnly: true
  778. }, {
  779. name: "$PERSISTENTVOLUMECLAIM_NAME",
  780. subPath: "repository",
  781. mountPath: "/opt/app-root/src",
  782. readOnly: false
  783. }]
  784. }
  785. EOF
  786. }
  787. provide() {
  788. [ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] && [ -n "$4" ] \
  789. && true || false
  790. local REPOSITORY_PATH="$1"
  791. local PUBLISHER="$2"
  792. local BUILD_ARCH="$3"
  793. local BUILD_OS="$4"
  794. # TODO: This code have to be refactored when generated images will have real realease
  795. # packages without -dev. This could be achieved by adding an aditional step to the
  796. # release process where the packages are extracted from the packaged images and
  797. # re-packaged in a new tick release packaged image.
  798. mkdir -p "/opt/app-root/src/$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS"
  799. for BASE_EXTENSION_TAR in "/var/lib/postgresql/extensions/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/"*.tar
  800. do
  801. BASE_EXTENSION_TAR_NAME="${BASE_EXTENSION_TAR##*/}"
  802. for EXTENSION_TAR in "$BASE_EXTENSION_TAR" "${BASE_EXTENSION_TAR%-dev.tar}.tar"
  803. do
  804. EXTENSION_TAR_NAME="${EXTENSION_TAR##*/}"
  805. if [ "${BASE_EXTENSION_TAR_NAME%-dev.tar}.tar" = "$EXTENSION_TAR_NAME" ]
  806. then
  807. if [ "$BASE_EXTENSION_TAR_NAME" = "${BASE_EXTENSION_TAR_NAME%-dev.tar}.tar" ]
  808. then
  809. continue
  810. fi
  811. if ! test -f "$EXTENSION_TAR"
  812. then
  813. rm -f "${EXTENSION_TAR%.*}.sha256" "${EXTENSION_TAR%.*}.tgz"
  814. tar xCf "${BASE_EXTENSION_TAR%/*}" "$BASE_EXTENSION_TAR"
  815. mv "${BASE_EXTENSION_TAR%.*}".sha256 "${EXTENSION_TAR%.*}".sha256
  816. mv "${BASE_EXTENSION_TAR%.*}".tgz "${EXTENSION_TAR%.*}".tgz
  817. tar cCf "${BASE_EXTENSION_TAR%/*}" "$EXTENSION_TAR" \
  818. "${EXTENSION_TAR_NAME%.*}.sha256" "${EXTENSION_TAR_NAME%.*}.tgz"
  819. fi
  820. rm -f "${EXTENSION_TAR%.*}.sha256" "${EXTENSION_TAR%.*}.tgz"
  821. fi
  822. rm -f "/opt/app-root/src/$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_TAR_NAME.tmp"
  823. ln -s "/proc/$$/root/$EXTENSION_TAR" \
  824. "/opt/app-root/src/$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_TAR_NAME.tmp"
  825. mv "/opt/app-root/src/$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_TAR_NAME.tmp" \
  826. "/opt/app-root/src/$REPOSITORY_PATH/$PUBLISHER/$BUILD_ARCH/$BUILD_OS/$EXTENSION_TAR_NAME"
  827. done
  828. done
  829. while true
  830. do
  831. sleep 300
  832. done
  833. }
  834. urlencode() {
  835. sed 's/\(.\)/\1\n/g' \
  836. | {
  837. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  838. NEWLINE="$(printf '\n')"
  839. while IFS="$NEWLINE" read -r C
  840. do
  841. case "$C" in
  842. [-_.~a-zA-Z0-9]) printf %c "$C" ;;
  843. "") printf %%0A ;;
  844. *) printf %%%02X "'$C'" ;;
  845. esac
  846. done
  847. }
  848. }
  849. urldecode() {
  850. sed 's/\(.\)/\1\n/g' \
  851. | {
  852. [ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
  853. NEWLINE="$(printf '\n')"
  854. CODE=
  855. while IFS="$NEWLINE" read -r C
  856. do
  857. case "$C" in
  858. \+)
  859. if [ -n "$CODE" ]
  860. then
  861. >&2 echo "Wrong code $CODE$C"
  862. exit 1
  863. fi
  864. printf ' '
  865. ;;
  866. %)
  867. if [ -n "$CODE" ]
  868. then
  869. >&2 echo "Wrong code $CODE$C"
  870. exit 1
  871. fi
  872. CODE='0x'
  873. ;;
  874. *)
  875. if [ -z "$CODE" ]
  876. then
  877. printf %c "$C"
  878. else
  879. CODE="$CODE$C"
  880. if [ -z "${CODE#0x??}" ]
  881. then
  882. # shellcheck disable=SC2059
  883. printf "$(printf '\\%03o' "$CODE")"
  884. CODE=
  885. fi
  886. fi
  887. ;;
  888. esac
  889. done
  890. }
  891. }
  892. try_function() {
  893. local E_UNSET=true
  894. if echo "$-" | grep -q e
  895. then
  896. E_UNSET=false
  897. fi
  898. "$E_UNSET" || set +e
  899. (set -e; "$@")
  900. EXIT_CODE="$?"
  901. "$E_UNSET" || set -e
  902. RESULT=false
  903. if [ "$EXIT_CODE" = 0 ]
  904. then
  905. RESULT=true
  906. fi
  907. }
  908. if [ -n "$1" ]
  909. then
  910. "$@"
  911. fi