fixup! [wip][feature] Add support for fsspec backends

.github/workflows/tests.yml

@@ -132,7 +132,7 @@ jobs:
         # zstd, may also call external binaries depending on how libarchive was compiled!
         # https://github.com/libarchive/libarchive/blob/ad5a0b542c027883d7069f6844045e6788c7d70c/libarchive/
         #   archive_read_support_filter_lrzip.c#L68
-        sudo apt-get -y install libfuse2 fuse3 bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev
+        sudo apt-get -y install libfuse2 fuse3 bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev ruby-webrick
 
     - name: Install Dependencies For Unreleased Python Versions (Linux)
       if: >

tests/runtests.sh

@@ -1797,26 +1797,55 @@ checkURLProtocolFile()
 
 checkURLProtocolHTTP()
 {
-    local pid
+    local pid mountPoint protocol port
+    mountPoint=$( mktemp -d )
+    protocol='http'
+    port=8000
+
     # Failed alternatives to set up a test HTTP server:
     #     python3 -m http.server -b 127.0.0.1 8000 &  # Does not support range requests
     #     python3 -m RangeHTTPServer -b 127.0.0.1 8000 &  # Client has spurious errors every 5th test or so with this.
     #         TODO Debug this... Bug could be in fsspec/implementations/http.py, aiohttp, RangeHTTPServer, ...
     #     sudo apt install busybox-static
     #     busybox httpd -f -p 8000 &  # Does not support range requests.
     # sudo apt install ruby-webrick
-    ruby -run -e httpd . --port 8000 --bind-address=127.0.0.1 &>/dev/null &
+    if ! command -v ruby &>/dev/null; then
+        echo "Ruby not found"
+        return 1
+    fi
+    ruby -run -e httpd --help
+    ruby -run -e httpd . --port $port --bind-address=127.0.0.1 &
     pid=$!
     sleep 1
 
-    checkFileInTAR 'http://127.0.0.1:8000/tests/single-file.tar' bar d3b07384d113edec49eaa6238ad5ff00 ||
-        returnError "$LINENO" 'Failed to read from HTTP server'
-    checkFileInTAR 'http://127.0.0.1:8000/tests/' single-file.tar 1a28538854d1884e4415cb9bfb7a2ad8 ||
-        returnError "$LINENO" 'Failed to read from HTTP server folder'
-    checkFileInTAR 'http://127.0.0.1:8000/tests' single-file.tar 1a28538854d1884e4415cb9bfb7a2ad8 ||
-        returnError "$LINENO" 'Failed to read from HTTP server folder'
+    for (( i=0; i<10; ++i )); do wget 127.0.0.1:$port; sleep 1; done
 
-    kill $pid
+    archive="$protocol://127.0.0.1:$port/tests/single-file.tar"
+    runAndCheckRatarmount -c -f -d 3 "$archive" "$mountPoint" &
+    while ! mountpoint -q -- "$mountPoint"; do sleep 1s; done
+    echo "Check access to $archive"
+    verifyCheckSum "$mountPoint" "bar" "$archive" d3b07384d113edec49eaa6238ad5ff00 ||
+        returnError "$LINENO" 'Checksum mismatches!'
+    funmount "$mountPoint"
+
+    archive="$protocol://127.0.0.1:$port/tests/"
+    runAndCheckRatarmount -c -f -d 3 "$archive" "$mountPoint" &
+    while ! mountpoint -q -- "$mountPoint"; do sleep 1s; done
+    echo "Check access to $archive"
+    verifyCheckSum "$mountPoint" 'single-file.tar' "$archive" 1a28538854d1884e4415cb9bfb7a2ad8 ||
+        returnError "$LINENO" 'Checksum mismatches!'
+    funmount "$mountPoint"
+
+    archive="$protocol://127.0.0.1:$port/tests"
+    runAndCheckRatarmount -c -f -d 3 "$archive" "$mountPoint" &
+    while ! mountpoint -q -- "$mountPoint"; do sleep 1s; done
+    echo "Check access to $archive"
+    verifyCheckSum "$mountPoint" 'single-file.tar' "$archive" 1a28538854d1884e4415cb9bfb7a2ad8 ||
+        returnError "$LINENO" 'Checksum mismatches!'
+    funmount "$mountPoint"
+
+    kill $pid &>/dev/null
+    rmdir "$mountPoint"
 }
 
 
@@ -1857,7 +1886,7 @@ killRogueSSH()
 
 checkURLProtocolSSH()
 {
-    local pid fingerprint publicKey mountPoint port
+    local pid fingerprint publicKey mountPoint port file
     # rm -f ssh_host_key; ssh-keygen -q -N "" -C "" -t ed25519 -f ssh_host_key
     cat <<EOF > ssh_host_key
 -----BEGIN OPENSSH PRIVATE KEY-----
@@ -1871,11 +1900,17 @@ EOF
     # Only works on server. Also not hashed in not in known_hosts format.
     #fingerprint=$( ssh-keygen -lf ssh_host_key )
     fingerprint=$( ssh-keyscan -H -p 8022 127.0.0.1 2>/dev/null )
-    'grep' -q -F "$fingerprint" ~/.ssh/known_hosts || echo "$fingerprint" >> ~/.ssh/known_hosts
+    file='~/.ssh/known_hosts'
+    if [[ ! -f "$file" ]] || ! 'grep' -q -F "$fingerprint" "$file"; then
+        echo "$fingerprint" >> "$file"
+    fi
 
     [[ -f ~/.ssh/id_ed25519 ]] || ssh-keygen -q -N "" -t ed25519 -f ~/.ssh/id_ed25519
     publicKey=$( cat ~/.ssh/id_ed25519.pub )
-    'grep' -q -F "$publicKey" ssh_user_ca || echo "$publicKey" >> ssh_user_ca
+    file='ssh_user_ca'
+    if [[ ! -f "$file" ]] || ! 'grep' -q -F "$publicKey" "$file"; then
+        echo "$publicKey" >> "$file"
+    fi
 
     killRogueSSH
     port=8022
@@ -1887,7 +1922,7 @@ EOF
     mountPoint=$( mktemp -d )
 
     archive="ssh://127.0.0.1:$port/tests/single-file.tar"
-    runAndCheckRatarmount -c -f "$archive" "$mountPoint" &
+    runAndCheckRatarmount -d 3 -c -f "$archive" "$mountPoint" &
     while ! mountpoint -q -- "$mountPoint"; do sleep 1s; done
     echo "Check access to $archive"
     verifyCheckSum "$mountPoint" "bar" "$archive" d3b07384d113edec49eaa6238ad5ff00 ||
@@ -1912,6 +1947,7 @@ EOF
 
     kill $pid
     killRogueSSH
+    rmdir "$mountPoint"
 }
 
 
@@ -2100,6 +2136,9 @@ EOF
 
 checkRemoteSupport()
 {
+    #checkURLProtocolSSH
+    checkURLProtocolHTTP
+    exit 1
     # Some implementations of fsspec. See e.g. this list:
     # https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations
 
@@ -2108,7 +2147,7 @@ checkRemoteSupport()
     checkURLProtocolGithub || returnError 'Failed github:// check'
     checkURLProtocolFTP || returnError 'Failed ftp:// check'
 
-    #checkURLProtocolHTTP  # TODO
+    checkURLProtocolHTTP || returnError 'Failed http:// check'
     #checkURLProtocolS3   # TODO suddenly broken again ...
     checkURLProtocolSSH || returnError 'Failed ssh:// check'