package main
import (
"context"
"craftdesign.group/quarry/internal/chat"
"crypto/tls"
"errors"
"log"
"net"
"net/url"
"strings"
"time"
"craftdesign.group/quarry/internal/children"
"craftdesign.group/quarry/internal/lease"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
)
// A repository's mirror to a Fossil server (RFC 0022, Fossil remote):
// the node holding the store's lease runs `zit push --once` to the
// remote with the stored login, and records what happened in words.
//
// The triggers and the one-run-at-a-time rule are the Git mirror's
// (mirrorRuns): after a fold, which is every way the store changes, and
// the retry tick. Unlike a Git push, a Fossil push carries everything
// (wiki, tickets, forum, every branch), so a kick never skips a run
// because trunk's tip is unchanged; the tick is what is selective.
// fossilMirrorTimeout bounds one run, for the reason gitMirrorTimeout
// does.
const fossilMirrorTimeout = 30 * time.Minute
// fossilMirrors is the Fossil mirror's run state.
type fossilMirrors struct {
mirrorRuns
// host, when set, refuses a remote whose host the forge may not
// reach, checked again at each push because a name can come to
// resolve elsewhere after it was accepted.
host func(ctx context.Context, host string) error
}
// fossilMirrorKick runs name's mirror now, or once more after the run
// in flight. Never blocks the caller.
func (rg *registrar) fossilMirrorKick(name string) {
if rg.odb == nil || !rg.jobs.store(name) {
return
}
rg.fossilMirror.kick(name, rg.fossilMirrorRun)
}
// fossilMirrorRetry kicks the mirrors that have never pushed, and those
// whose last push failed and whose backoff has passed. One that has
// pushed and not failed since waits for the next fold.
func (rg *registrar) fossilMirrorRetry() {
if rg.odb == nil {
return
}
repos, err := rg.odb.FossilMirrorRepos()
if err != nil {
log.Printf("fossil mirror: list: %v", err)
return
}
at := time.Now()
for _, name := range repos {
m, on, err := rg.odb.FossilMirrorOf(name)
if err != nil || !on || !fossilMirrorDue(m.PushedAt, m.ErrAt, m.Fails, at) {
continue
}
rg.fossilMirrorKick(name)
}
}
// fossilMirrorDue says whether the retry tick runs a mirror: one that
// has never succeeded, or whose last attempt failed, once the backoff
// has passed.
func fossilMirrorDue(pushedAt, errAt string, fails int, at time.Time) bool {
if pushedAt != "" && errAt == "" {
return false
}
return !fossilMirrorBackingOff(errAt, fails, at)
}
// fossilMirrorBackingOff says the last attempt failed and its wait has
// not passed.
func fossilMirrorBackingOff(errAt string, fails int, at time.Time) bool {
if errAt == "" {
return false
}
failed, err := time.Parse(time.RFC3339, errAt)
return err == nil && at.Before(failed.Add(gitMirrorBackoff(fails)))
}
// fossilPush is zitcli.Client.FossilPush, named so a run can be handed
// one.
type fossilPush func(ctx context.Context, remote, user string, env []string) (string, error)
// fossilMirrorRun is one pass for one repository: nothing when there is
// no mirror or it is paused (private, taken down); otherwise one push.
//
// A kick obeys the backoff the retry tick does: after a failure the
// next attempt waits, so a wrong password is not tried after every
// fold. Changing the settings clears the failure, so a corrected
// password is tried at once.
//
// The run works from the one row it reads here: remote, user and
// generation. The password is read later only if that generation still
// stands, and the outcome is recorded only if it does, so a change made
// while the push runs is never sent the old destination's password nor
// credited with its result.
func (rg *registrar) fossilMirrorRun(name string, at time.Time) {
m, on, err := rg.odb.FossilMirrorOf(name)
if err != nil || !on {
return
}
if fossilMirrorBackingOff(m.ErrAt, m.Fails, at) {
return
}
// Paused, not failed, and read from the database closed on error,
// for the reasons gitMirrorRun gives.
if priv, err := rg.odb.PrivateRepos(); err != nil || priv[name] {
return
}
if _, held, err := rg.odb.TakedownOf(name); err != nil || held {
return
}
repo, ok := rg.deps.Repos.Get(name)
if !ok || repo.Zit == nil || repo.Index == nil || repo.Guard == nil {
return
}
var pushed string
if tip, ok, err := repo.Index.TipOf("trunk"); err == nil && ok {
pushed = tip.Tip
}
var why string
var decided bool
if err := repo.Guard(func(tok lease.Token) error {
why, decided = rg.fossilMirrorPush(name, m.Remote, m.User, m.Gen, func(ctx context.Context, remote, user string, env []string) (string, error) {
return repo.Zit.FossilPush(tok, ctx, remote, user, env)
})
// A push cut short by the lease going elsewhere says nothing
// about the mirror: the next holder pushes without a backoff.
if tok.Err() != nil {
decided = false
}
return nil
}); err != nil || !decided {
return
}
if why == "" {
if err := rg.odb.RecordFossilMirrorPush(name, m.Gen, pushed); err != nil {
log.Printf("fossil mirror: %s: record: %v", name, err)
}
return
}
if err := rg.odb.RecordFossilMirrorFailure(name, m.Gen, why); err != nil {
log.Printf("fossil mirror: %s: record: %v", name, err)
}
}
// fossilMirrorPush reads the password for the length of one push and
// runs it. why is "" on success, else the failure in words; decided is
// false when the push was stopped from outside (the forge stopping),
// which says nothing about the mirror and records nothing.
//
// The password goes to zit in its environment only, never its
// arguments, and is not logged: what is logged is zit's own output,
// which never carries it.
func (rg *registrar) fossilMirrorPush(name, remote, user string, gen int64, push fossilPush) (why string, decided bool) {
ctx, cancel := context.WithTimeout(context.Background(), fossilMirrorTimeout)
defer cancel()
if check := rg.fossilMirror.host; check != nil {
if u, err := url.Parse(remote); err == nil {
if err := check(ctx, u.Hostname()); err != nil {
log.Printf("fossil mirror: %s: %v", name, err)
return "the forge may not push to that host: it is a private address", true
}
}
}
password, err := rg.odb.FossilMirrorPassword(name, gen)
if errors.Is(err, oltp.ErrFossilMirrorChanged) {
// Changed since this run read it: the next run reads the new one.
return "", false
}
if err != nil {
log.Printf("fossil mirror: %s: password: %v", name, err)
return "the saved password could not be read; stop mirroring and mirror again to give it again", true
}
out, err := push(ctx, remote, user, []string{"ZIT_PASSWORD=" + password})
switch {
case err == nil:
return "", true
case errors.Is(err, ops.ErrStopping), children.Default.IsStopping():
return "", false
case ctx.Err() != nil:
return "the push did not finish within " + fossilMirrorTimeout.String(), true
}
log.Printf("fossil mirror: %s: %v: %s", name, err, lastLines(out, 6))
// zit says "cannot connect" for a refused certificate as for a
// closed port (zit 0.14.4 prints nothing more), so the two are told
// apart here, by asking the host for its certificate.
if strings.Contains(out, "cannot connect") && certificateRefused(ctx, remote, rg.fossilMirror.host != nil) {
return fossilCertificateWords, true
}
return fossilMirrorWords(out), true
}
// fossilCertificateWords is the failure when the server's certificate
// could not be verified: self-signed, expired, for another name, or from
// an authority this machine does not trust.
const fossilCertificateWords = "the server's certificate could not be verified"
// certificateRefused reports whether a TLS handshake with remote's host
// fails on the certificate. False for an address that is not https, a
// host that cannot be reached, and a handshake that succeeds.
func certificateRefused(ctx context.Context, remote string, publicOnly bool) bool {
u, err := url.Parse(remote)
if err != nil || u.Scheme != "https" || u.Hostname() == "" {
return false
}
port := u.Port()
if port == "" {
port = "443"
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
// This is the forge's own connection to a host a repository admin
// named, so it dials only addresses it has checked, as the chat
// bridge does: asking again by name could be answered differently.
dial := (&net.Dialer{Timeout: 15 * time.Second}).DialContext
if publicOnly {
dial = chat.DialPublic
}
raw, err := dial(ctx, "tcp", net.JoinHostPort(u.Hostname(), port))
if err != nil {
return false
}
defer raw.Close()
err = tls.Client(raw, &tls.Config{ServerName: u.Hostname()}).HandshakeContext(ctx)
var bad *tls.CertificateVerificationError
return errors.As(err, &bad)
}
// fossilMirrorWords puts a refused push in words a repository admin can
// act on, from what zit said.
func fossilMirrorWords(out string) string {
has := func(subs ...string) bool {
for _, s := range subs {
if strings.Contains(out, s) {
return true
}
}
return false
}
switch {
case has("project code", "project-code"):
return "the remote is not a copy of this repository: its project code differs"
case has("login failed"):
// A wrong user or password, and a server whose project code
// differs (the login is signed with it), look the same to zit.
return "the server refused the login: check the user and password, and that the remote is a copy of this repository (the same project code)"
case has("not authorized", "not authorised"):
return "the login is not allowed to push on that server: it needs the capability to check in"
case has("cannot connect", "no such host", "connection refused", "timed out", "network is unreachable", "dial tcp"):
return "the server could not be reached"
}
if l := strings.TrimRight(lastLines(out, 1), ". "); l != "" {
return "the push failed: " + l
}
return "the push failed"
}