// uvclient.go is RFC 0020 rung 6's wire half: quarry as a fossil
// CLIENT of an upstream's unversioned files, the other side of the
// cards unversioned.go answers. A mirror's import round sends
// `pragma uv-hash` and `uvgimme` and reads `uvigot` and `uvfile` back;
// it never sends a uvfile, because a mirror only reads.
//
// The request is built as fossil's client builds it: src/http.c
// (http_exchange, http_build_login_card, by way of buildLoginBody),
// src/sha1.c (sha1_shared_secret) and src/xfer.c (client_sync: the
// client-version pragma, the line of randomness the login's nonce is
// taken over).
package xfer
import (
"bytes"
"compress/zlib"
"context"
"crypto/rand"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"craftdesign.group/quarry/internal/uv"
)
// uvClientVersion is the release this client announces in
// `pragma client-version`: fossil 2.28's number. A server sends a
// SHA3 hash only to a client at 2.0 or later (send_unversioned_file),
// and a client that announces nothing is taken as older.
const uvClientVersion = "22800"
// MaxUVReply bounds one decoded reply whatever UVUpstream asks for:
// the 256 MiB the sync front reads of a request. fossil's own budget is 5 MB
// a reply, but its first file is sent whole whatever its size.
const MaxUVReply = 256 << 20
// Each exchange's timeouts: the headers within UVHeaderTimeout, the
// whole reply within UVExchangeTimeout, so a stalled upstream costs one
// exchange and not the round.
const (
UVHeaderTimeout = 30 * time.Second
UVExchangeTimeout = 5 * time.Minute
)
// UVUpstream is one upstream as the import round speaks to it.
type UVUpstream struct {
// URL is the upstream's sync address, as fossil's client posts to
// it. A user and password in it are dropped: the round signs with
// User and Password, or as nobody.
URL string
// User and Password sign each request with a login card; an empty
// User is nobody, who sends none.
User, Password string
// ProjectCode is the upstream's, which the login's shared secret
// is derived from. A mirror holds the same one, learned when it was
// cloned.
ProjectCode string
// Client makes the request; nil is one with UVHeaderTimeout. Each
// exchange is also bounded by UVExchangeTimeout.
Client *http.Client
// MaxReply bounds one decoded reply; 0, or more than MaxUVReply, is
// MaxUVReply.
MaxReply int64
}
// UVReply is what an upstream answered.
type UVReply struct {
// Pragma is "uv-push-ok" or "uv-pull-only", empty when the request's
// uv-hash drew neither: the hashes matched, or the login may not
// read the files.
Pragma string
// Catalog is the uvigot cards in order; a tombstone is Deleted.
Catalog []uv.File
// Files is the uvfile cards in order.
Files []UVReplyFile
// Err is the first error card, in words.
Err string
// Bytes is the reply's size on the wire.
Bytes int64
}
// UVReplyFile is one uvfile card. Bad is fossil's words for a card that
// was framed but cannot be taken (a name quarry refuses, a bad hash),
// and File.Name is then the name as sent.
type UVReplyFile struct {
File uv.File
Content []byte
// Omitted is flag 0x0004: the content was left out.
Omitted bool
Bad string
}
// Exchange sends cards to the upstream and reads its reply. The cards
// go after `pragma client-version` and before a comment line of
// randomness, so no two requests sign the same bytes, as client_sync
// does; with a User the login card goes first and its nonce covers
// everything after it.
func (u UVUpstream) Exchange(ctx context.Context, cards string) (UVReply, error) {
target, err := url.Parse(u.URL)
if err != nil {
return UVReply{}, fmt.Errorf("upstream address: %v", err)
}
// Go would send a user in the URL as HTTP Basic. That is the DAG
// pull's identity, not the round's, and fossil's sync reads only
// the login card.
target.User, target.Fragment = nil, ""
var random [20]byte
if _, err := rand.Read(random[:]); err != nil {
return UVReply{}, err
}
payload := []byte("pragma client-version " + uvClientVersion + "\n" + cards + "# " + hex.EncodeToString(random[:]) + "\n")
body := payload
if u.User != "" {
body = buildLoginBody(u.ProjectCode, u.User, u.Password, string(payload))
}
// Posted to /xfer under the address, as FetchUsers posts: fossil's
// server answers a sync there whatever page the address names.
target.Path = strings.TrimSuffix(target.Path, "/") + "/xfer"
ctx, cancel := context.WithTimeout(ctx, UVExchangeTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), bytes.NewReader(encodeContainer(body)))
if err != nil {
return UVReply{}, err
}
req.Header.Set("Content-Type", "application/x-fossil")
client := u.Client
if client == nil {
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.ResponseHeaderTimeout = UVHeaderTimeout
client = &http.Client{Transport: tr}
}
// A redirect is not followed: Go turns a 301 or 302 into a GET,
// which a fossil server answers with a page, and following one
// silently would send the next login card wherever it pointed.
c := *client
c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
resp, err := c.Do(req)
if err != nil {
return UVReply{}, err
}
defer resp.Body.Close()
if resp.StatusCode/100 == 3 {
return UVReply{}, fmt.Errorf("the upstream redirects to %s; set the mirror's address to where it points", resp.Header.Get("Location"))
}
if resp.StatusCode != http.StatusOK {
return UVReply{}, fmt.Errorf("the upstream answered %s", resp.Status)
}
limit := u.MaxReply
if limit <= 0 || limit > MaxUVReply {
limit = MaxUVReply
}
// Inflated as it arrives, so the compressed reply is never held
// beside the plain one. The length the container claims is the
// upstream's word: over the limit it is refused before anything is
// inflated, and the inflate stops at it either way.
wire := &countReader{r: io.LimitReader(resp.Body, limit)}
var head [4]byte
if _, err := io.ReadFull(wire, head[:]); err != nil {
return UVReply{}, errNotFossil
}
claim := int64(binary.BigEndian.Uint32(head[:]))
if claim > limit {
return UVReply{}, fmt.Errorf("the upstream's reply would be %d bytes, over %d", claim, limit)
}
zr, err := zlib.NewReader(wire)
if err != nil {
return UVReply{}, errNotFossil
}
defer zr.Close()
plain, err := io.ReadAll(io.LimitReader(zr, claim+1))
if err != nil || int64(len(plain)) != claim {
if ctx.Err() != nil {
return UVReply{}, fmt.Errorf("the upstream's reply did not arrive within %s", UVExchangeTimeout)
}
return UVReply{}, errNotFossil
}
r, err := scanReply(plain)
r.Bytes = wire.n
return r, err
}
// errNotFossil is a reply that is not a sync message: an HTML page, a
// proxy's error, a truncated or oversized container.
var errNotFossil = errors.New("the upstream's reply is not a fossil sync message; is the address a fossil repository?")
// countReader counts the bytes read through it.
type countReader struct {
r io.Reader
n int64
}
func (c *countReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += int64(n)
return n, err
}
// scanReply reads an upstream's reply for the unversioned cards, framed
// as scanRequest frames a request: a uvfile's payload, and a file or
// config card's, is skipped by its byte count and never read as cards.
// What cannot be framed refuses the whole reply. cfile is refused too:
// a server sends one only to a client that asked for artifacts, and
// this client never does.
func scanReply(body []byte) (UVReply, error) {
var r UVReply
pos := 0
for pos < len(body) {
start := pos
line := body[pos:]
next := len(body)
if k := bytes.IndexByte(line, '\n'); k >= 0 {
line, next = line[:k], pos+k+1
}
pos = next
f := cardTokens(line)
if len(f) == 0 {
continue
}
switch f[0] {
case "pragma":
if len(f) == 2 && (f[1] == "uv-push-ok" || f[1] == "uv-pull-only") && r.Pragma == "" {
r.Pragma = f[1]
}
case "uvigot":
x, ok := uvigotCard(f)
if !ok {
return r, fmt.Errorf("malformed uvigot line: %.200s", line)
}
r.Catalog = append(r.Catalog, x)
case "uvfile":
c, ok := frameUVFile(f, body, start, pos)
if !ok {
return r, fmt.Errorf("malformed uvfile line: %.200s", line)
}
pos = c.end
in, why := uvFileCard(c)
if why != "" {
in.Name = f[1]
}
r.Files = append(r.Files, UVReplyFile{File: in, Content: c.payload, Omitted: c.omitted(), Bad: why})
case "error":
if r.Err == "" {
r.Err = strings.Join(strings.Fields(defossilize(strings.Join(f[1:], " "))), " ")
}
case "file":
size, ok := payloadSize(f, 3, 4)
if !ok || size > len(body)-pos {
return r, errors.New("malformed file card")
}
pos += size
case "config":
size, ok := payloadSize(f, 3, 3)
if !ok || size >= len(body)-pos || body[pos+size] != '\n' {
return r, errors.New("malformed config card")
}
pos += size + 1
case "cfile":
return r, errors.New("the upstream sent a cfile card, which this client never asks for")
}
}
return r, nil
}
// uvigotCard reads "uvigot NAME MTIME HASH SIZE" as the client half of
// xfer.c reads it: MTIME and SIZE not negative, HASH a hash or "-" for
// a tombstone. The name is kept as sent; the round decides whether it
// can carry it.
func uvigotCard(f []string) (uv.File, bool) {
if len(f) != 5 {
return uv.File{}, false
}
mtime, err := strconv.ParseInt(f[2], 10, 64)
if err != nil || mtime < 0 {
return uv.File{}, false
}
size, err := strconv.ParseInt(f[4], 10, 64)
if err != nil || size < 0 {
return uv.File{}, false
}
if f[3] == "-" {
return uv.File{Name: f[1], Mtime: mtime, Deleted: true}, true
}
if !isHname(f[3]) {
return uv.File{}, false
}
return uv.File{Name: f[1], Mtime: mtime, Hash: f[3], Size: size}, true
}