package main
import (
"context"
"crypto/tls"
"errors"
"log"
"net"
"net/url"
"strings"
"time"
"craftdesign.group/quarry/internal/chat"
"craftdesign.group/quarry/internal/children"
"craftdesign.group/quarry/internal/lease"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
)
// A repository's mirror to a Fossil server (RFC 0022, Fossil remote):
// the node holding the store's lease runs `zit push --once` to the
// remote with the stored login, and records what happened in words.
//
// The triggers and the one-run-at-a-time rule are the Git mirror's
// (mirrorRuns): after a fold, which is every way the store changes, and
// the retry tick. Unlike a Git push, a Fossil push carries everything
// (wiki, tickets, forum, every branch), so a kick never skips a run
// because trunk's tip is unchanged; the tick is what is selective.
// fossilMirrorTimeout bounds one run, for the reason gitMirrorTimeout
// does.
const fossilMirrorTimeout = 30 * time.Minute
// fossilMirrors is the Fossil mirror's run state.
type fossilMirrors struct {
mirrorRuns
// resolve answers a remote's host with the addresses it may be
// pushed to: chat.PublicAddrs, which refuses a private one, unless
// the forge was started with --allow-private-bridges, when
// chat.ResolveHost. Nil is the latter. Asked once per push; the
// address chosen is pinned in zit's arguments, so the name is never
// looked up again and a changed answer cannot matter.
resolve func(ctx context.Context, host string) ([]net.IPAddr, error)
}
// fossilMirrorKick runs name's mirror now, or once more after the run
// in flight. Never blocks the caller.
func (rg *registrar) fossilMirrorKick(name string) {
if rg.odb == nil || !rg.jobs.store(name) {
return
}
rg.fossilMirror.kick(name, rg.fossilMirrorRun)
}
// fossilMirrorRetry kicks the mirrors that have never pushed, and those
// whose last push failed and whose backoff has passed. One that has
// pushed and not failed since waits for the next fold.
func (rg *registrar) fossilMirrorRetry() {
if rg.odb == nil {
return
}
repos, err := rg.odb.FossilMirrorRepos()
if err != nil {
log.Printf("fossil mirror: list: %v", err)
return
}
at := time.Now()
for _, name := range repos {
m, on, err := rg.odb.FossilMirrorOf(name)
if err != nil || !on || !fossilMirrorDue(m.PushedAt, m.ErrAt, m.Fails, at) {
continue
}
rg.fossilMirrorKick(name)
}
}
// fossilMirrorDue says whether the retry tick runs a mirror: one that
// has never succeeded, or whose last attempt failed, once the backoff
// has passed.
func fossilMirrorDue(pushedAt, errAt string, fails int, at time.Time) bool {
if pushedAt != "" && errAt == "" {
return false
}
return !fossilMirrorBackingOff(errAt, fails, at)
}
// fossilMirrorBackingOff says the last attempt failed and its wait has
// not passed.
func fossilMirrorBackingOff(errAt string, fails int, at time.Time) bool {
if errAt == "" {
return false
}
failed, err := time.Parse(time.RFC3339, errAt)
return err == nil && at.Before(failed.Add(gitMirrorBackoff(fails)))
}
// fossilPush is zitcli.Client.FossilPush, named so a run can be handed
// one.
type fossilPush func(ctx context.Context, remote, user string, resolve, env []string) (string, error)
// fossilMirrorRun is one pass for one repository: nothing when there is
// no mirror or it is paused (private, taken down); otherwise one push.
//
// A kick obeys the backoff the retry tick does: after a failure the
// next attempt waits, so a wrong password is not tried after every
// fold. Changing the settings clears the failure, so a corrected
// password is tried at once.
//
// The run works from the one row it reads here: remote, user and
// generation. The password is read later only if that generation still
// stands, and the outcome is recorded only if it does, so a change made
// while the push runs is never sent the old destination's password nor
// credited with its result.
func (rg *registrar) fossilMirrorRun(name string, at time.Time) {
m, on, err := rg.odb.FossilMirrorOf(name)
if err != nil || !on {
return
}
if fossilMirrorBackingOff(m.ErrAt, m.Fails, at) {
return
}
// Paused, not failed, and read from the database closed on error,
// for the reasons gitMirrorRun gives.
if priv, err := rg.odb.PrivateRepos(); err != nil || priv[name] {
return
}
if _, held, err := rg.odb.TakedownOf(name); err != nil || held {
return
}
repo, ok := rg.deps.Repos.Get(name)
if !ok || repo.Zit == nil || repo.Index == nil || repo.Guard == nil {
return
}
var pushed string
if tip, ok, err := repo.Index.TipOf("trunk"); err == nil && ok {
pushed = tip.Tip
}
var why string
var decided bool
if err := repo.Guard(func(tok lease.Token) error {
why, decided = rg.fossilMirrorPush(name, m.Remote, m.User, m.Gen, func(ctx context.Context, remote, user string, resolve, env []string) (string, error) {
return repo.Zit.FossilPush(tok, ctx, remote, user, resolve, env)
})
// A push cut short by the lease going elsewhere says nothing
// about the mirror: the next holder pushes without a backoff.
if tok.Err() != nil {
decided = false
}
return nil
}); err != nil || !decided {
return
}
if why == "" {
if err := rg.odb.RecordFossilMirrorPush(name, m.Gen, pushed); err != nil {
log.Printf("fossil mirror: %s: record: %v", name, err)
}
return
}
if err := rg.odb.RecordFossilMirrorFailure(name, m.Gen, why); err != nil {
log.Printf("fossil mirror: %s: record: %v", name, err)
}
}
// fossilMirrorPush reads the password for the length of one push and
// runs it. why is "" on success, else the failure in words; decided is
// false when the push was stopped from outside (the forge stopping),
// which says nothing about the mirror and records nothing.
//
// The password goes to zit in its environment only, never its
// arguments, and is not logged: what is logged is zit's own output,
// which never carries it.
func (rg *registrar) fossilMirrorPush(name, remote, user string, gen int64, push fossilPush) (why string, decided bool) {
ctx, cancel := context.WithTimeout(context.Background(), fossilMirrorTimeout)
defer cancel()
password, err := rg.odb.FossilMirrorPassword(name, gen)
if errors.Is(err, oltp.ErrFossilMirrorChanged) {
// Changed since this run read it: the next run reads the new one.
return "", false
}
if err != nil {
log.Printf("fossil mirror: %s: password: %v", name, err)
return "the saved password could not be read; stop mirroring and mirror again to give it again", true
}
// Resolved and vetted once, here: zit is handed the address, never
// the chance to look the name up.
u, perr := url.Parse(remote)
if perr != nil || u.Hostname() == "" {
return "the remote address is not one a push can use", true
}
resolve := rg.fossilMirror.resolve
if resolve == nil {
resolve = chat.ResolveHost
}
addrs, err := resolve(ctx, u.Hostname())
if err != nil || len(addrs) == 0 {
log.Printf("fossil mirror: %s: %v", name, err)
if err != nil && strings.Contains(err.Error(), "refusing") {
return "the forge may not push to that host: it is a private address", true
}
return "the server's name could not be resolved", true
}
env := []string{"ZIT_PASSWORD=" + password}
// zit takes one address per host and port. The first the resolver
// gave is used; if zit cannot connect to it, the next is tried, up
// to fossilMirrorAddrs, each one an address that was vetted.
var out, via string
for i, a := range addrs {
if i == fossilMirrorAddrs {
break
}
via = a.IP.String()
out, err = push(ctx, remote, user, fossilPins(u, a.IP), env)
if err == nil || !strings.Contains(out, "cannot connect") || ctx.Err() != nil {
break
}
}
switch {
case err == nil:
return "", true
case errors.Is(err, ops.ErrStopping), children.Default.IsStopping():
return "", false
case ctx.Err() != nil:
return "the push did not finish within " + fossilMirrorTimeout.String(), true
}
log.Printf("fossil mirror: %s: %v: %s", name, err, lastLines(out, 6))
// zit says "cannot connect" for a refused certificate as for a
// closed port, so the two are told apart here, by asking the host
// for its certificate at the address the push used.
if strings.Contains(out, "cannot connect") && certificateRefused(ctx, remote, via) {
return fossilCertificateWords, true
}
return fossilMirrorWords(out, u.Hostname()), true
}
// fossilMirrorAddrs is how many of a host's addresses a push tries.
const fossilMirrorAddrs = 3
// fossilPins is the --resolve entries that pin remote's host to addr,
// for the port the URL names, or the scheme's. An http address on port
// 80 may be upgraded to https on 443, which zit then needs an entry
// for. None for a host that is itself an address: there is nothing to
// look up, and "HOST:PORT:ADDR" cannot be told from an IPv6 literal.
func fossilPins(u *url.URL, addr net.IP) []string {
host := u.Hostname()
if net.ParseIP(host) != nil {
return nil
}
port := u.Port()
if port == "" {
port = map[string]string{"http": "80", "https": "443"}[u.Scheme]
}
pins := []string{host + ":" + port + ":" + addr.String()}
if u.Scheme == "http" && port == "80" {
pins = append(pins, host+":443:"+addr.String())
}
return pins
}
// fossilCertificateWords is the failure when the server's certificate
// could not be verified: self-signed, expired, for another name, or from
// an authority this machine does not trust.
const fossilCertificateWords = "the server's certificate could not be verified"
// certificateRefused reports whether a TLS handshake with remote's host
// fails on the certificate. False for an address that is not https, a
// host that cannot be reached, and a handshake that succeeds. It
// connects to via, the address the push used and the forge vetted,
// never to the name, which could be answered differently now.
func certificateRefused(ctx context.Context, remote, via string) bool {
u, err := url.Parse(remote)
if err != nil || u.Scheme != "https" || u.Hostname() == "" || via == "" {
return false
}
port := u.Port()
if port == "" {
port = "443"
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
raw, err := (&net.Dialer{Timeout: 15 * time.Second}).DialContext(ctx, "tcp", net.JoinHostPort(via, port))
if err != nil {
return false
}
defer raw.Close()
err = tls.Client(raw, &tls.Config{ServerName: u.Hostname()}).HandshakeContext(ctx)
var bad *tls.CertificateVerificationError
return errors.As(err, &bad)
}
// fossilMirrorWords puts a refused push in words a repository admin can
// act on, from what zit said.
func fossilMirrorWords(out, host string) string {
has := func(subs ...string) bool {
for _, s := range subs {
if strings.Contains(out, s) {
return true
}
}
return false
}
switch {
case has("redirected to"):
// zit names the target, which can carry a credential: only the
// remote's own host is named here.
return "the server at " + host + " redirected the push elsewhere, which the forge does not follow: give the address it redirects to"
case has("no --resolve entry"):
return "the forge could not pin the server's address for the push: an internal fault"
case has("project code", "project-code"):
return "the remote is not a copy of this repository: its project code differs"
case has("login failed"):
// A wrong user or password, and a server whose project code
// differs (the login is signed with it), look the same to zit.
return "the server refused the login: check the user and password, and that the remote is a copy of this repository (the same project code)"
case has("not authorized", "not authorised"):
return "the login is not allowed to push on that server: it needs the capability to check in"
case has("cannot connect", "no such host", "connection refused", "timed out", "network is unreachable", "dial tcp"):
return "the server could not be reached"
}
if l := strings.TrimRight(lastLines(out, 1), ". "); l != "" {
return "the push failed: " + l
}
return "the push failed"
}