From: "Eric Wong (Contractor, The Linux Foundation)" <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/4] v2writable: refer to git each repository as "epoch"
Date: Thu, 5 Apr 2018 21:45:26 +0000 [thread overview]
Message-ID: <20180405214528.19403-3-e@80x24.org> (raw)
In-Reply-To: <20180405214528.19403-1-e@80x24.org>
This hopefully helps for people who try to understand
this design.
---
lib/PublicInbox/V2Writable.pm | 54 +++++++++++++++++++++----------------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index ea116f1..b858dcc 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -69,7 +69,7 @@ sub new {
xpfx => $xpfx,
over => PublicInbox::OverIdxFork->new("$xpfx/over.sqlite3"),
lock_path => "$dir/inbox.lock",
- # limit each repo to 1GB or so
+ # limit each git repo (epoch) to 1GB or so
rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
last_commit => [], # git repo -> commit
};
@@ -81,9 +81,9 @@ sub init_inbox {
my ($self, $parallel) = @_;
$self->{parallel} = $parallel;
$self->idx_init;
- my $max_git = -1;
- git_dir_latest($self, \$max_git);
- $self->git_init($max_git >= 0 ? $max_git : 0);
+ my $epoch_max = -1;
+ git_dir_latest($self, \$epoch_max);
+ $self->git_init($epoch_max >= 0 ? $epoch_max : 0);
$self->done;
}
@@ -115,7 +115,7 @@ sub add {
my $nparts = $self->{partitions};
my $part = $num % $nparts;
- $self->{last_commit}->[$self->{max_git}] = $cmt;
+ $self->{last_commit}->[$self->{epoch_max}] = $cmt;
my $idx = $self->idx_part($part);
$idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime);
my $n = $self->{transact_bytes} += $len;
@@ -242,7 +242,7 @@ sub purge_oids {
$self->done;
my $pfx = "$self->{-inbox}->{mainrepo}/git";
my $purges = [];
- foreach my $i (0..$self->{max_git}) {
+ foreach my $i (0..$self->{epoch_max}) {
my $git = PublicInbox::Git->new("$pfx/$i.git");
my $im = $self->import_init($git, 0, 1);
$purges->[$i] = $im->purge_oids($purge);
@@ -315,7 +315,7 @@ sub remove_internal {
if (defined $mark) {
my $cmt = $im->get_mark($mark);
- $self->{last_commit}->[$self->{max_git}] = $cmt;
+ $self->{last_commit}->[$self->{epoch_max}] = $cmt;
}
if ($purge && scalar keys %$purge) {
return purge_oids($self, $purge);
@@ -347,9 +347,9 @@ sub last_commit_part ($$;$) {
sub set_last_commits ($) {
my ($self) = @_;
- defined(my $max_git = $self->{max_git}) or return;
+ defined(my $epoch_max = $self->{epoch_max}) or return;
my $last_commit = $self->{last_commit};
- foreach my $i (0..$max_git) {
+ foreach my $i (0..$epoch_max) {
defined(my $cmt = $last_commit->[$i]) or next;
$last_commit->[$i] = undef;
last_commit_part($self, $i, $cmt);
@@ -430,9 +430,9 @@ sub barrier {
}
sub git_init {
- my ($self, $new) = @_;
+ my ($self, $epoch) = @_;
my $pfx = "$self->{-inbox}->{mainrepo}/git";
- my $git_dir = "$pfx/$new.git";
+ my $git_dir = "$pfx/$epoch.git";
my @cmd = (qw(git init --bare -q), $git_dir);
PublicInbox::Import::run_die(\@cmd);
@@ -450,7 +450,7 @@ sub git_init {
PublicInbox::Import::run_die(\@cmd);
my $alt = "$all/objects/info/alternates";
- my $new_obj_dir = "../../git/$new.git/objects";
+ my $new_obj_dir = "../../git/$epoch.git/objects";
my %alts;
if (-e $alt) {
open(my $fh, '<', $alt) or die "open < $alt: $!\n";
@@ -491,26 +491,26 @@ sub importer {
$im->done;
$self->barrier(1);
$im = undef;
- my $git_dir = $self->git_init(++$self->{max_git});
+ my $git_dir = $self->git_init(++$self->{epoch_max});
my $git = PublicInbox::Git->new($git_dir);
return $self->import_init($git, 0);
}
}
- my $new = 0;
+ my $epoch = 0;
my $max;
my $latest = git_dir_latest($self, \$max);
if (defined $latest) {
my $git = PublicInbox::Git->new($latest);
my $packed_bytes = $git->packed_bytes;
if ($packed_bytes >= $self->{rotate_bytes}) {
- $new = $max + 1;
+ $epoch = $max + 1;
} else {
- $self->{max_git} = $max;
+ $self->{epoch_max} = $max;
return $self->import_init($git, $packed_bytes);
}
}
- $self->{max_git} = $new;
- $latest = $self->git_init($new);
+ $self->{epoch_max} = $epoch;
+ $latest = $self->git_init($epoch);
$self->import_init(PublicInbox::Git->new($latest), 0);
}
@@ -690,9 +690,9 @@ sub update_last_commit {
sub git_dir_n ($$) { "$_[0]->{-inbox}->{mainrepo}/git/$_[1].git" }
sub last_commits {
- my ($self, $max_git) = @_;
+ my ($self, $epoch_max) = @_;
my $heads = [];
- for (my $i = $max_git; $i >= 0; $i--) {
+ for (my $i = $epoch_max; $i >= 0; $i--) {
$heads->[$i] = last_commit_part($self, $i);
}
$heads;
@@ -710,10 +710,10 @@ sub is_ancestor ($$$) {
}
sub index_prepare {
- my ($self, $opts, $max_git, $ranges) = @_;
+ my ($self, $opts, $epoch_max, $ranges) = @_;
my $regen_max = 0;
my $head = $self->{-inbox}->{ref_head} || 'refs/heads/master';
- for (my $i = $max_git; $i >= 0; $i--) {
+ for (my $i = $epoch_max; $i >= 0; $i--) {
die "already indexing!\n" if $self->{index_pipe};
my $git_dir = git_dir_n($self, $i);
-d $git_dir or next; # missing parts are fine
@@ -822,15 +822,15 @@ sub index_sync {
my ($self, $opts) = @_;
$opts ||= {};
my $ibx = $self->{-inbox};
- my $max_git;
- my $latest = git_dir_latest($self, \$max_git);
+ my $epoch_max;
+ my $latest = git_dir_latest($self, \$epoch_max);
return unless defined $latest;
$self->idx_init; # acquire lock
my $mm_tmp = $self->{mm}->tmp_clone;
- my $ranges = $opts->{reindex} ? [] : $self->last_commits($max_git);
+ my $ranges = $opts->{reindex} ? [] : $self->last_commits($epoch_max);
my ($min, $max) = $mm_tmp->minmax;
- my $regen = $self->index_prepare($opts, $max_git, $ranges);
+ my $regen = $self->index_prepare($opts, $epoch_max, $ranges);
$$regen += $max if $max;
my $D = {};
my @cmd = qw(log --raw -r --pretty=tformat:%h
@@ -838,7 +838,7 @@ sub index_sync {
# work backwards through history
my $last_commit = [];
- for (my $i = $max_git; $i >= 0; $i--) {
+ for (my $i = $epoch_max; $i >= 0; $i--) {
my $git_dir = git_dir_n($self, $i);
die "already reindexing!\n" if delete $self->{reindex_pipe};
-d $git_dir or next; # missing parts are fine
--
EW
next prev parent reply other threads:[~2018-04-05 21:45 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-04-05 21:45 [PATCH 0/4] a few more v2 updates Eric Wong (Contractor, The Linux Foundation)
2018-04-05 21:45 ` [PATCH 1/4] v2writable: allow tracking parallel versions Eric Wong (Contractor, The Linux Foundation)
2018-04-05 21:45 ` Eric Wong (Contractor, The Linux Foundation) [this message]
2018-04-05 21:45 ` [PATCH 3/4] over: use only supported and safe SQLite APIs Eric Wong (Contractor, The Linux Foundation)
2018-04-05 21:45 ` [PATCH 4/4] search: index and allow searching by date-time Eric Wong (Contractor, The Linux Foundation)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180405214528.19403-3-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).