librsync  2.3.3
scoop.c
Go to the documentation of this file.
1/*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*-
2 *
3 * librsync -- the library for network deltas
4 *
5 * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1 of
10 * the License, or (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this program; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 */
21
22 /*=
23 | To walk on water you've gotta sink
24 | in the ice.
25 | -- Shihad, `The General Electric'.
26 */
27
28/** \file scoop.c
29 * This file deals with readahead from caller-supplied buffers.
30 *
31 * Many functions require a certain minimum amount of contiguous input data to
32 * do their processing. For example, to calculate a strong checksum of a block
33 * we need at least a block of input.
34 *
35 * Since we put the buffers completely under the control of the caller, we
36 * can't count on ever getting this much data all in one go. We can't simply
37 * wait, because the caller might have a smaller buffer than we require and so
38 * we'll never get it.
39 *
40 * Stream input data is used directly if there is sufficient data to satisfy
41 * the readhead requests, otherwise it is copied and accumulated into an
42 * internal buffer until there is enough. This means for large input buffers we
43 * can leave a "tail" of unprocessed data in the input buffer, and only consume
44 * all the data if it was too small and start accumulating into the internal
45 * buffer. Provided the input buffers always have enough data we avoid copying
46 * into the internal buffer at all.
47 *
48 * \todo We probably know a maximum amount of data that can be scooped up, so
49 * we could just avoid dynamic allocation. However that can't be fixed at
50 * compile time, because when generating a delta it needs to be large enough to
51 * hold one full block. Perhaps we can set it up when the job is allocated? It
52 * would be kind of nice to not do any memory allocation after startup, as
53 * bzlib does this. */
54
55#include <assert.h>
56#include <stdlib.h>
57#include <string.h>
58#include "librsync.h"
59#include "job.h"
60#include "scoop.h"
61#include "trace.h"
62#include "util.h"
63
64/** Try to accept a from the input buffer to get LEN bytes in the scoop. */
65static inline void rs_scoop_input(rs_job_t *job, size_t len)
66{
67 rs_buffers_t *stream = job->stream;
68 size_t tocopy;
69
70 assert(len > job->scoop_avail);
71
72 if (job->scoop_alloc < len) {
73 /* Need to allocate a larger scoop. */
74 rs_byte_t *newbuf;
75 size_t newsize;
76 for (newsize = 64; newsize < len; newsize <<= 1) ;
77 newbuf = rs_alloc(newsize, "scoop buffer");
78 if (job->scoop_avail)
79 memcpy(newbuf, job->scoop_next, job->scoop_avail);
80 if (job->scoop_buf)
81 free(job->scoop_buf);
82 job->scoop_buf = job->scoop_next = newbuf;
83 rs_trace("resized scoop buffer to " FMT_SIZE " bytes from " FMT_SIZE "",
84 newsize, job->scoop_alloc);
85 job->scoop_alloc = newsize;
86 } else if (job->scoop_buf + job->scoop_alloc < job->scoop_next + len) {
87 /* Move existing data to the front of the scoop. */
88 rs_trace("moving scoop " FMT_SIZE " bytes to reuse " FMT_SIZE " bytes",
89 job->scoop_avail, (size_t)(job->scoop_next - job->scoop_buf));
90 memmove(job->scoop_buf, job->scoop_next, job->scoop_avail);
91 job->scoop_next = job->scoop_buf;
92 }
93 /* take as much input as is available, to give up to LEN bytes in the
94 scoop. */
95 tocopy = len - job->scoop_avail;
96 if (tocopy > stream->avail_in)
97 tocopy = stream->avail_in;
98 assert(job->scoop_next + tocopy + job->scoop_avail <=
99 job->scoop_buf + job->scoop_alloc);
100
101 memcpy(job->scoop_next + job->scoop_avail, stream->next_in, tocopy);
102 rs_trace("accepted " FMT_SIZE " bytes from input to scoop", tocopy);
103 job->scoop_avail += tocopy;
104 stream->next_in += tocopy;
105 stream->avail_in -= tocopy;
106}
107
108/** Advance the input cursor forward \p len bytes.
109 *
110 * This is used after doing readahead, when you decide you want to keep it. \p
111 * len must be no more than the amount of available data, so you can't cheat.
112 *
113 * So when creating a delta, we require one block of readahead. But after
114 * examining that block, we might decide to advance over all of it (if there is
115 * a match), or just one byte (if not). */
116void rs_scoop_advance(rs_job_t *job, size_t len)
117{
118 rs_buffers_t *stream = job->stream;
119
120 /* It never makes sense to advance over a mixture of bytes from the scoop
121 and input, because you couldn't possibly have looked at them all at the
122 same time. */
123 if (job->scoop_avail) {
124 /* reading from the scoop buffer */
125 rs_trace("advance over " FMT_SIZE " bytes from scoop", len);
126 assert(len <= job->scoop_avail);
127 job->scoop_avail -= len;
128 job->scoop_next += len;
129 } else {
130 rs_trace("advance over " FMT_SIZE " bytes from input buffer", len);
131 assert(len <= stream->avail_in);
132 stream->avail_in -= len;
133 stream->next_in += len;
134 }
135}
136
137/** Read from scoop without advancing.
138 *
139 * Ask for LEN bytes of input from the stream. If that much data is available,
140 * then return a pointer to it in PTR, advance the stream input pointer over
141 * the data, and return RS_DONE. If there's not enough data, then accept
142 * whatever is there into a buffer, advance over it, and return RS_BLOCKED.
143 *
144 * The data is not actually removed from the input, so this function lets you
145 * do readahead. If you want to keep any of the data, you should also call
146 * rs_scoop_advance() to skip over it. */
147rs_result rs_scoop_readahead(rs_job_t *job, size_t len, void **ptr)
148{
149 rs_buffers_t *stream = job->stream;
150 rs_job_check(job);
151
152 if (!job->scoop_avail && stream->avail_in >= len) {
153 /* The scoop is empty and there's enough data in the input. */
154 *ptr = stream->next_in;
155 rs_trace("got " FMT_SIZE " bytes direct from input", len);
156 return RS_DONE;
157 } else if (job->scoop_avail < len && stream->avail_in) {
158 /* There is not enough data in the scoop. */
159 rs_trace("scoop has less than " FMT_SIZE " bytes, scooping from "
160 FMT_SIZE " input bytes", len, stream->avail_in);
161 rs_scoop_input(job, len);
162 }
163 if (job->scoop_avail >= len) {
164 /* There is enough data in the scoop now. */
165 rs_trace("scoop has at least " FMT_SIZE " bytes, this is enough",
166 job->scoop_avail);
167 *ptr = job->scoop_next;
168 return RS_DONE;
169 } else if (stream->eof_in) {
170 /* Not enough input data and at EOF. */
171 rs_trace("reached end of input stream");
172 return RS_INPUT_ENDED;
173 } else {
174 /* Not enough input data yet. */
175 rs_trace("blocked with insufficient input data");
176 return RS_BLOCKED;
177 }
178}
179
180/** Read LEN bytes if possible, and remove them from the input scoop.
181 *
182 * \param *job An rs_job_t pointer to the job instance.
183 *
184 * \param len The length of the data in the ptr buffer.
185 *
186 * \param **ptr will be updated to point to a read-only buffer holding the
187 * data, if enough is available.
188 *
189 * \return RS_DONE if there was enough data, RS_BLOCKED if there was not enough
190 * data yet, or RS_INPUT_ENDED if there was not enough data and at EOF. */
191rs_result rs_scoop_read(rs_job_t *job, size_t len, void **ptr)
192{
193 rs_result result;
194
195 result = rs_scoop_readahead(job, len, ptr);
196 if (result == RS_DONE)
197 rs_scoop_advance(job, len);
198 return result;
199}
200
201/** Read whatever data remains in the input stream.
202 *
203 * \param *job The rs_job_t instance the job instance.
204 *
205 * \param *len will be updated to the length of the available data.
206 *
207 * \param **ptr will point at the available data.
208 *
209 * \return RS_DONE if there was data, RS_INPUT_ENDED if there was no data and
210 * at EOF, RS_BLOCKED if there was no data and not at EOF. */
211rs_result rs_scoop_read_rest(rs_job_t *job, size_t *len, void **ptr)
212{
213 *len = rs_scoop_avail(job);
214 if (*len)
215 return rs_scoop_read(job, *len, ptr);
216 else if (job->stream->eof_in)
217 return RS_INPUT_ENDED;
218 else
219 return RS_BLOCKED;
220}
Generic state-machine interface.
#define rs_job_check(job)
Assert that a job is valid.
Definition: job.h:130
Public header for librsync.
rs_result
Return codes from nonblocking rsync operations.
Definition: librsync.h:180
@ RS_DONE
Completed successfully.
Definition: librsync.h:181
@ RS_INPUT_ENDED
Unexpected end of input file, perhaps due to a truncated file or dropped network connection.
Definition: librsync.h:190
@ RS_BLOCKED
Blocked waiting for more data.
Definition: librsync.h:182
rs_result rs_scoop_readahead(rs_job_t *job, size_t len, void **ptr)
Read from scoop without advancing.
Definition: scoop.c:147
static void rs_scoop_input(rs_job_t *job, size_t len)
Try to accept a from the input buffer to get LEN bytes in the scoop.
Definition: scoop.c:65
rs_result rs_scoop_read(rs_job_t *job, size_t len, void **ptr)
Read LEN bytes if possible, and remove them from the input scoop.
Definition: scoop.c:191
void rs_scoop_advance(rs_job_t *job, size_t len)
Advance the input cursor forward len bytes.
Definition: scoop.c:116
rs_result rs_scoop_read_rest(rs_job_t *job, size_t *len, void **ptr)
Read whatever data remains in the input stream.
Definition: scoop.c:211
Manage librsync streams of IO.
Description of input and output buffers.
Definition: librsync.h:328
char * next_in
Next input byte.
Definition: librsync.h:334
size_t avail_in
Number of bytes available at next_in.
Definition: librsync.h:342
int eof_in
True if there is no more data after this.
Definition: librsync.h:345
The contents of this structure are private.
Definition: job.h:47
rs_byte_t * scoop_buf
Buffer of data in the scoop.
Definition: job.h:97
size_t scoop_alloc
The buffer allocation size.
Definition: job.h:99
rs_byte_t * scoop_next
The next data pointer.
Definition: job.h:98
size_t scoop_avail
The amount of data available.
Definition: job.h:100
logging functions.
Misc utility functions used by librsync.