source: code/trunk/vendor/github.com/andybalholm/brotli/encode.go@ 145

Last change on this file since 145 was 145, checked in by Izuru Yakumo, 22 months ago

Updated the Makefile and vendored depedencies

Signed-off-by: Izuru Yakumo <yakumo.izuru@…>
File size: 40.1 KB
RevLine 
[145]1package brotli
2
3import (
4 "io"
5 "math"
6)
7
8/* Copyright 2016 Google Inc. All Rights Reserved.
9
10 Distributed under MIT license.
11 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
12*/
13
14/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
15const minWindowBits = 10
16
17/**
18 * Maximal value for ::BROTLI_PARAM_LGWIN parameter.
19 *
20 * @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
21 */
22const maxWindowBits = 24
23
24/**
25 * Maximal value for ::BROTLI_PARAM_LGWIN parameter
26 * in "Large Window Brotli" (32-bit).
27 */
28const largeMaxWindowBits = 30
29
30/** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
31const minInputBlockBits = 16
32
33/** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
34const maxInputBlockBits = 24
35
36/** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
37const minQuality = 0
38
39/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
40const maxQuality = 11
41
42/** Options for ::BROTLI_PARAM_MODE parameter. */
43const (
44 modeGeneric = 0
45 modeText = 1
46 modeFont = 2
47)
48
49/** Default value for ::BROTLI_PARAM_QUALITY parameter. */
50const defaultQuality = 11
51
52/** Default value for ::BROTLI_PARAM_LGWIN parameter. */
53const defaultWindow = 22
54
55/** Default value for ::BROTLI_PARAM_MODE parameter. */
56const defaultMode = modeGeneric
57
58/** Operations that can be performed by streaming encoder. */
59const (
60 operationProcess = 0
61 operationFlush = 1
62 operationFinish = 2
63 operationEmitMetadata = 3
64)
65
66const (
67 streamProcessing = 0
68 streamFlushRequested = 1
69 streamFinished = 2
70 streamMetadataHead = 3
71 streamMetadataBody = 4
72)
73
74type Writer struct {
75 dst io.Writer
76 options WriterOptions
77 err error
78
79 params encoderParams
80 hasher_ hasherHandle
81 input_pos_ uint64
82 ringbuffer_ ringBuffer
83 commands []command
84 num_literals_ uint
85 last_insert_len_ uint
86 last_flush_pos_ uint64
87 last_processed_pos_ uint64
88 dist_cache_ [numDistanceShortCodes]int
89 saved_dist_cache_ [4]int
90 last_bytes_ uint16
91 last_bytes_bits_ byte
92 prev_byte_ byte
93 prev_byte2_ byte
94 storage []byte
95 small_table_ [1 << 10]int
96 large_table_ []int
97 large_table_size_ uint
98 cmd_depths_ [128]byte
99 cmd_bits_ [128]uint16
100 cmd_code_ [512]byte
101 cmd_code_numbits_ uint
102 command_buf_ []uint32
103 literal_buf_ []byte
104 tiny_buf_ struct {
105 u64 [2]uint64
106 u8 [16]byte
107 }
108 remaining_metadata_bytes_ uint32
109 stream_state_ int
110 is_last_block_emitted_ bool
111 is_initialized_ bool
112}
113
114func inputBlockSize(s *Writer) uint {
115 return uint(1) << uint(s.params.lgblock)
116}
117
118func unprocessedInputSize(s *Writer) uint64 {
119 return s.input_pos_ - s.last_processed_pos_
120}
121
122func remainingInputBlockSize(s *Writer) uint {
123 var delta uint64 = unprocessedInputSize(s)
124 var block_size uint = inputBlockSize(s)
125 if delta >= uint64(block_size) {
126 return 0
127 }
128 return block_size - uint(delta)
129}
130
131/* Wraps 64-bit input position to 32-bit ring-buffer position preserving
132 "not-a-first-lap" feature. */
133func wrapPosition(position uint64) uint32 {
134 var result uint32 = uint32(position)
135 var gb uint64 = position >> 30
136 if gb > 2 {
137 /* Wrap every 2GiB; The first 3GB are continuous. */
138 result = result&((1<<30)-1) | (uint32((gb-1)&1)+1)<<30
139 }
140
141 return result
142}
143
144func (s *Writer) getStorage(size int) []byte {
145 if len(s.storage) < size {
146 s.storage = make([]byte, size)
147 }
148
149 return s.storage
150}
151
152func hashTableSize(max_table_size uint, input_size uint) uint {
153 var htsize uint = 256
154 for htsize < max_table_size && htsize < input_size {
155 htsize <<= 1
156 }
157
158 return htsize
159}
160
161func getHashTable(s *Writer, quality int, input_size uint, table_size *uint) []int {
162 var max_table_size uint = maxHashTableSize(quality)
163 var htsize uint = hashTableSize(max_table_size, input_size)
164 /* Use smaller hash table when input.size() is smaller, since we
165 fill the table, incurring O(hash table size) overhead for
166 compression, and if the input is short, we won't need that
167 many hash table entries anyway. */
168
169 var table []int
170 assert(max_table_size >= 256)
171 if quality == fastOnePassCompressionQuality {
172 /* Only odd shifts are supported by fast-one-pass. */
173 if htsize&0xAAAAA == 0 {
174 htsize <<= 1
175 }
176 }
177
178 if htsize <= uint(len(s.small_table_)) {
179 table = s.small_table_[:]
180 } else {
181 if htsize > s.large_table_size_ {
182 s.large_table_size_ = htsize
183 s.large_table_ = nil
184 s.large_table_ = make([]int, htsize)
185 }
186
187 table = s.large_table_
188 }
189
190 *table_size = htsize
191 for i := 0; i < int(htsize); i++ {
192 table[i] = 0
193 }
194 return table
195}
196
197func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) {
198 if large_window {
199 *last_bytes = uint16((lgwin&0x3F)<<8 | 0x11)
200 *last_bytes_bits = 14
201 } else {
202 if lgwin == 16 {
203 *last_bytes = 0
204 *last_bytes_bits = 1
205 } else if lgwin == 17 {
206 *last_bytes = 1
207 *last_bytes_bits = 7
208 } else if lgwin > 17 {
209 *last_bytes = uint16((lgwin-17)<<1 | 0x01)
210 *last_bytes_bits = 4
211 } else {
212 *last_bytes = uint16((lgwin-8)<<4 | 0x01)
213 *last_bytes_bits = 7
214 }
215 }
216}
217
218/* Decide about the context map based on the ability of the prediction
219 ability of the previous byte UTF8-prefix on the next byte. The
220 prediction ability is calculated as Shannon entropy. Here we need
221 Shannon entropy instead of 'BitsEntropy' since the prefix will be
222 encoded with the remaining 6 bits of the following byte, and
223 BitsEntropy will assume that symbol to be stored alone using Huffman
224 coding. */
225
226var kStaticContextMapContinuation = [64]uint32{
227 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
228 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
229 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
230 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
231}
232var kStaticContextMapSimpleUTF8 = [64]uint32{
233 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
234 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
235 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
236 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
237}
238
239func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) {
240 var monogram_histo = [3]uint32{0}
241 var two_prefix_histo = [6]uint32{0}
242 var total uint
243 var i uint
244 var dummy uint
245 var entropy [4]float64
246 for i = 0; i < 9; i++ {
247 monogram_histo[i%3] += bigram_histo[i]
248 two_prefix_histo[i%6] += bigram_histo[i]
249 }
250
251 entropy[1] = shannonEntropy(monogram_histo[:], 3, &dummy)
252 entropy[2] = (shannonEntropy(two_prefix_histo[:], 3, &dummy) + shannonEntropy(two_prefix_histo[3:], 3, &dummy))
253 entropy[3] = 0
254 for i = 0; i < 3; i++ {
255 entropy[3] += shannonEntropy(bigram_histo[3*i:], 3, &dummy)
256 }
257
258 total = uint(monogram_histo[0] + monogram_histo[1] + monogram_histo[2])
259 assert(total != 0)
260 entropy[0] = 1.0 / float64(total)
261 entropy[1] *= entropy[0]
262 entropy[2] *= entropy[0]
263 entropy[3] *= entropy[0]
264
265 if quality < minQualityForHqContextModeling {
266 /* 3 context models is a bit slower, don't use it at lower qualities. */
267 entropy[3] = entropy[1] * 10
268 }
269
270 /* If expected savings by symbol are less than 0.2 bits, skip the
271 context modeling -- in exchange for faster decoding speed. */
272 if entropy[1]-entropy[2] < 0.2 && entropy[1]-entropy[3] < 0.2 {
273 *num_literal_contexts = 1
274 } else if entropy[2]-entropy[3] < 0.02 {
275 *num_literal_contexts = 2
276 *literal_context_map = kStaticContextMapSimpleUTF8[:]
277 } else {
278 *num_literal_contexts = 3
279 *literal_context_map = kStaticContextMapContinuation[:]
280 }
281}
282
283/* Decide if we want to use a more complex static context map containing 13
284 context values, based on the entropy reduction of histograms over the
285 first 5 bits of literals. */
286
287var kStaticContextMapComplexUTF8 = [64]uint32{
288 11, 11, 12, 12, /* 0 special */
289 0, 0, 0, 0, /* 4 lf */
290 1, 1, 9, 9, /* 8 space */
291 2, 2, 2, 2, /* !, first after space/lf and after something else. */
292 1, 1, 1, 1, /* " */
293 8, 3, 3, 3, /* % */
294 1, 1, 1, 1, /* ({[ */
295 2, 2, 2, 2, /* }]) */
296 8, 4, 4, 4, /* :; */
297 8, 7, 4, 4, /* . */
298 8, 0, 0, 0, /* > */
299 3, 3, 3, 3, /* [0..9] */
300 5, 5, 10, 5, /* [A-Z] */
301 5, 5, 10, 5,
302 6, 6, 6, 6, /* [a-z] */
303 6, 6, 6, 6,
304}
305
306func shouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool {
307 /* Try the more complex static context map only for long data. */
308 if size_hint < 1<<20 {
309 return false
310 } else {
311 var end_pos uint = start_pos + length
312 var combined_histo = [32]uint32{0}
313 var context_histo = [13][32]uint32{[32]uint32{0}}
314 var total uint32 = 0
315 var entropy [3]float64
316 var dummy uint
317 var i uint
318 var utf8_lut contextLUT = getContextLUT(contextUTF8)
319 /* To make entropy calculations faster and to fit on the stack, we collect
320 histograms over the 5 most significant bits of literals. One histogram
321 without context and 13 additional histograms for each context value. */
322 for ; start_pos+64 <= end_pos; start_pos += 4096 {
323 var stride_end_pos uint = start_pos + 64
324 var prev2 byte = input[start_pos&mask]
325 var prev1 byte = input[(start_pos+1)&mask]
326 var pos uint
327
328 /* To make the analysis of the data faster we only examine 64 byte long
329 strides at every 4kB intervals. */
330 for pos = start_pos + 2; pos < stride_end_pos; pos++ {
331 var literal byte = input[pos&mask]
332 var context byte = byte(kStaticContextMapComplexUTF8[getContext(prev1, prev2, utf8_lut)])
333 total++
334 combined_histo[literal>>3]++
335 context_histo[context][literal>>3]++
336 prev2 = prev1
337 prev1 = literal
338 }
339 }
340
341 entropy[1] = shannonEntropy(combined_histo[:], 32, &dummy)
342 entropy[2] = 0
343 for i = 0; i < 13; i++ {
344 entropy[2] += shannonEntropy(context_histo[i][0:], 32, &dummy)
345 }
346
347 entropy[0] = 1.0 / float64(total)
348 entropy[1] *= entropy[0]
349 entropy[2] *= entropy[0]
350
351 /* The triggering heuristics below were tuned by compressing the individual
352 files of the silesia corpus. If we skip this kind of context modeling
353 for not very well compressible input (i.e. entropy using context modeling
354 is 60% of maximal entropy) or if expected savings by symbol are less
355 than 0.2 bits, then in every case when it triggers, the final compression
356 ratio is improved. Note however that this heuristics might be too strict
357 for some cases and could be tuned further. */
358 if entropy[2] > 3.0 || entropy[1]-entropy[2] < 0.2 {
359 return false
360 } else {
361 *num_literal_contexts = 13
362 *literal_context_map = kStaticContextMapComplexUTF8[:]
363 return true
364 }
365 }
366}
367
368func decideOverLiteralContextModeling(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) {
369 if quality < minQualityForContextModeling || length < 64 {
370 return
371 } else if shouldUseComplexStaticContextMap(input, start_pos, length, mask, quality, size_hint, num_literal_contexts, literal_context_map) {
372 } else /* Context map was already set, nothing else to do. */
373 {
374 var end_pos uint = start_pos + length
375 /* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
376 UTF8 data faster we only examine 64 byte long strides at every 4kB
377 intervals. */
378
379 var bigram_prefix_histo = [9]uint32{0}
380 for ; start_pos+64 <= end_pos; start_pos += 4096 {
381 var lut = [4]int{0, 0, 1, 2}
382 var stride_end_pos uint = start_pos + 64
383 var prev int = lut[input[start_pos&mask]>>6] * 3
384 var pos uint
385 for pos = start_pos + 1; pos < stride_end_pos; pos++ {
386 var literal byte = input[pos&mask]
387 bigram_prefix_histo[prev+lut[literal>>6]]++
388 prev = lut[literal>>6] * 3
389 }
390 }
391
392 chooseContextMap(quality, bigram_prefix_histo[0:], num_literal_contexts, literal_context_map)
393 }
394}
395
396func shouldCompress_encode(data []byte, mask uint, last_flush_pos uint64, bytes uint, num_literals uint, num_commands uint) bool {
397 /* TODO: find more precise minimal block overhead. */
398 if bytes <= 2 {
399 return false
400 }
401 if num_commands < (bytes>>8)+2 {
402 if float64(num_literals) > 0.99*float64(bytes) {
403 var literal_histo = [256]uint32{0}
404 const kSampleRate uint32 = 13
405 const kMinEntropy float64 = 7.92
406 var bit_cost_threshold float64 = float64(bytes) * kMinEntropy / float64(kSampleRate)
407 var t uint = uint((uint32(bytes) + kSampleRate - 1) / kSampleRate)
408 var pos uint32 = uint32(last_flush_pos)
409 var i uint
410 for i = 0; i < t; i++ {
411 literal_histo[data[pos&uint32(mask)]]++
412 pos += kSampleRate
413 }
414
415 if bitsEntropy(literal_histo[:], 256) > bit_cost_threshold {
416 return false
417 }
418 }
419 }
420
421 return true
422}
423
424/* Chooses the literal context mode for a metablock */
425func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, length uint) int {
426 /* We only do the computation for the option of something else than
427 CONTEXT_UTF8 for the highest qualities */
428 if params.quality >= minQualityForHqBlockSplitting && !isMostlyUTF8(data, pos, mask, length, kMinUTF8Ratio) {
429 return contextSigned
430 }
431
432 return contextUTF8
433}
434
435func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
436 var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
437 var last_bytes uint16
438 var last_bytes_bits byte
439 var literal_context_lut contextLUT = getContextLUT(literal_context_mode)
440 var block_params encoderParams = *params
441
442 if bytes == 0 {
443 /* Write the ISLAST and ISEMPTY bits. */
444 writeBits(2, 3, storage_ix, storage)
445
446 *storage_ix = (*storage_ix + 7) &^ 7
447 return
448 }
449
450 if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) {
451 /* Restore the distance cache, as its last update by
452 CreateBackwardReferences is now unused. */
453 copy(dist_cache, saved_dist_cache[:4])
454
455 storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
456 return
457 }
458
459 assert(*storage_ix <= 14)
460 last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
461 last_bytes_bits = byte(*storage_ix)
462 if params.quality <= maxQualityForStaticEntropyCodes {
463 storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
464 } else if params.quality < minQualityForBlockSplit {
465 storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
466 } else {
467 mb := getMetaBlockSplit()
468 if params.quality < minQualityForHqBlockSplitting {
469 var num_literal_contexts uint = 1
470 var literal_context_map []uint32 = nil
471 if !params.disable_literal_context_modeling {
472 decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
473 }
474
475 buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, mb)
476 } else {
477 buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, mb)
478 }
479
480 if params.quality >= minQualityForOptimizeHistograms {
481 /* The number of distance symbols effectively used for distance
482 histograms. It might be less than distance alphabet size
483 for "Large Window Brotli" (32-bit). */
484 var num_effective_dist_codes uint32 = block_params.dist.alphabet_size
485 if num_effective_dist_codes > numHistogramDistanceSymbols {
486 num_effective_dist_codes = numHistogramDistanceSymbols
487 }
488
489 optimizeHistograms(num_effective_dist_codes, mb)
490 }
491
492 storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, storage_ix, storage)
493 freeMetaBlockSplit(mb)
494 }
495
496 if bytes+4 < *storage_ix>>3 {
497 /* Restore the distance cache and last byte. */
498 copy(dist_cache, saved_dist_cache[:4])
499
500 storage[0] = byte(last_bytes)
501 storage[1] = byte(last_bytes >> 8)
502 *storage_ix = uint(last_bytes_bits)
503 storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
504 }
505}
506
507func chooseDistanceParams(params *encoderParams) {
508 var distance_postfix_bits uint32 = 0
509 var num_direct_distance_codes uint32 = 0
510
511 if params.quality >= minQualityForNonzeroDistanceParams {
512 var ndirect_msb uint32
513 if params.mode == modeFont {
514 distance_postfix_bits = 1
515 num_direct_distance_codes = 12
516 } else {
517 distance_postfix_bits = params.dist.distance_postfix_bits
518 num_direct_distance_codes = params.dist.num_direct_distance_codes
519 }
520
521 ndirect_msb = (num_direct_distance_codes >> distance_postfix_bits) & 0x0F
522 if distance_postfix_bits > maxNpostfix || num_direct_distance_codes > maxNdirect || ndirect_msb<<distance_postfix_bits != num_direct_distance_codes {
523 distance_postfix_bits = 0
524 num_direct_distance_codes = 0
525 }
526 }
527
528 initDistanceParams(params, distance_postfix_bits, num_direct_distance_codes)
529}
530
531func ensureInitialized(s *Writer) bool {
532 if s.is_initialized_ {
533 return true
534 }
535
536 s.last_bytes_bits_ = 0
537 s.last_bytes_ = 0
538 s.remaining_metadata_bytes_ = math.MaxUint32
539
540 sanitizeParams(&s.params)
541 s.params.lgblock = computeLgBlock(&s.params)
542 chooseDistanceParams(&s.params)
543
544 ringBufferSetup(&s.params, &s.ringbuffer_)
545
546 /* Initialize last byte with stream header. */
547 {
548 var lgwin int = int(s.params.lgwin)
549 if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
550 lgwin = brotli_max_int(lgwin, 18)
551 }
552
553 encodeWindowBits(lgwin, s.params.large_window, &s.last_bytes_, &s.last_bytes_bits_)
554 }
555
556 if s.params.quality == fastOnePassCompressionQuality {
557 s.cmd_depths_ = [128]byte{
558 0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
559 0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7,
560 7, 7, 10, 10, 10, 10, 10, 10, 0, 4, 4, 5, 5, 5, 6, 6,
561 7, 8, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
562 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
563 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4,
564 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 10,
565 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
566 }
567 s.cmd_bits_ = [128]uint16{
568 0, 0, 8, 9, 3, 35, 7, 71,
569 39, 103, 23, 47, 175, 111, 239, 31,
570 0, 0, 0, 4, 12, 2, 10, 6,
571 13, 29, 11, 43, 27, 59, 87, 55,
572 15, 79, 319, 831, 191, 703, 447, 959,
573 0, 14, 1, 25, 5, 21, 19, 51,
574 119, 159, 95, 223, 479, 991, 63, 575,
575 127, 639, 383, 895, 255, 767, 511, 1023,
576 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
577 27, 59, 7, 39, 23, 55, 30, 1, 17, 9, 25, 5, 0, 8, 4, 12,
578 2, 10, 6, 21, 13, 29, 3, 19, 11, 15, 47, 31, 95, 63, 127, 255,
579 767, 2815, 1791, 3839, 511, 2559, 1535, 3583, 1023, 3071, 2047, 4095,
580 }
581 s.cmd_code_ = [512]byte{
582 0xff, 0x77, 0xd5, 0xbf, 0xe7, 0xde, 0xea, 0x9e, 0x51, 0x5d, 0xde, 0xc6,
583 0x70, 0x57, 0xbc, 0x58, 0x58, 0x58, 0xd8, 0xd8, 0x58, 0xd5, 0xcb, 0x8c,
584 0xea, 0xe0, 0xc3, 0x87, 0x1f, 0x83, 0xc1, 0x60, 0x1c, 0x67, 0xb2, 0xaa,
585 0x06, 0x83, 0xc1, 0x60, 0x30, 0x18, 0xcc, 0xa1, 0xce, 0x88, 0x54, 0x94,
586 0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00,
587 }
588 s.cmd_code_numbits_ = 448
589 }
590
591 s.is_initialized_ = true
592 return true
593}
594
595func encoderInitParams(params *encoderParams) {
596 params.mode = defaultMode
597 params.large_window = false
598 params.quality = defaultQuality
599 params.lgwin = defaultWindow
600 params.lgblock = 0
601 params.size_hint = 0
602 params.disable_literal_context_modeling = false
603 initEncoderDictionary(&params.dictionary)
604 params.dist.distance_postfix_bits = 0
605 params.dist.num_direct_distance_codes = 0
606 params.dist.alphabet_size = uint32(distanceAlphabetSize(0, 0, maxDistanceBits))
607 params.dist.max_distance = maxDistance
608}
609
610func encoderInitState(s *Writer) {
611 encoderInitParams(&s.params)
612 s.input_pos_ = 0
613 s.commands = s.commands[:0]
614 s.num_literals_ = 0
615 s.last_insert_len_ = 0
616 s.last_flush_pos_ = 0
617 s.last_processed_pos_ = 0
618 s.prev_byte_ = 0
619 s.prev_byte2_ = 0
620 if s.hasher_ != nil {
621 s.hasher_.Common().is_prepared_ = false
622 }
623 s.cmd_code_numbits_ = 0
624 s.stream_state_ = streamProcessing
625 s.is_last_block_emitted_ = false
626 s.is_initialized_ = false
627
628 ringBufferInit(&s.ringbuffer_)
629
630 /* Initialize distance cache. */
631 s.dist_cache_[0] = 4
632
633 s.dist_cache_[1] = 11
634 s.dist_cache_[2] = 15
635 s.dist_cache_[3] = 16
636
637 /* Save the state of the distance cache in case we need to restore it for
638 emitting an uncompressed block. */
639 copy(s.saved_dist_cache_[:], s.dist_cache_[:])
640}
641
642/*
643 Copies the given input data to the internal ring buffer of the compressor.
644 No processing of the data occurs at this time and this function can be
645 called multiple times before calling WriteBrotliData() to process the
646 accumulated input. At most input_block_size() bytes of input data can be
647 copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
648*/
649func copyInputToRingBuffer(s *Writer, input_size uint, input_buffer []byte) {
650 var ringbuffer_ *ringBuffer = &s.ringbuffer_
651 ringBufferWrite(input_buffer, input_size, ringbuffer_)
652 s.input_pos_ += uint64(input_size)
653
654 /* TL;DR: If needed, initialize 7 more bytes in the ring buffer to make the
655 hashing not depend on uninitialized data. This makes compression
656 deterministic and it prevents uninitialized memory warnings in Valgrind.
657 Even without erasing, the output would be valid (but nondeterministic).
658
659 Background information: The compressor stores short (at most 8 bytes)
660 substrings of the input already read in a hash table, and detects
661 repetitions by looking up such substrings in the hash table. If it
662 can find a substring, it checks whether the substring is really there
663 in the ring buffer (or it's just a hash collision). Should the hash
664 table become corrupt, this check makes sure that the output is
665 still valid, albeit the compression ratio would be bad.
666
667 The compressor populates the hash table from the ring buffer as it's
668 reading new bytes from the input. However, at the last few indexes of
669 the ring buffer, there are not enough bytes to build full-length
670 substrings from. Since the hash table always contains full-length
671 substrings, we erase with dummy zeros here to make sure that those
672 substrings will contain zeros at the end instead of uninitialized
673 data.
674
675 Please note that erasing is not necessary (because the
676 memory region is already initialized since he ring buffer
677 has a `tail' that holds a copy of the beginning,) so we
678 skip erasing if we have already gone around at least once in
679 the ring buffer.
680
681 Only clear during the first round of ring-buffer writes. On
682 subsequent rounds data in the ring-buffer would be affected. */
683 if ringbuffer_.pos_ <= ringbuffer_.mask_ {
684 /* This is the first time when the ring buffer is being written.
685 We clear 7 bytes just after the bytes that have been copied from
686 the input buffer.
687
688 The ring-buffer has a "tail" that holds a copy of the beginning,
689 but only once the ring buffer has been fully written once, i.e.,
690 pos <= mask. For the first time, we need to write values
691 in this tail (where index may be larger than mask), so that
692 we have exactly defined behavior and don't read uninitialized
693 memory. Due to performance reasons, hashing reads data using a
694 LOAD64, which can go 7 bytes beyond the bytes written in the
695 ring-buffer. */
696 for i := 0; i < int(7); i++ {
697 ringbuffer_.buffer_[ringbuffer_.pos_:][i] = 0
698 }
699 }
700}
701
702/* Marks all input as processed.
703 Returns true if position wrapping occurs. */
704func updateLastProcessedPos(s *Writer) bool {
705 var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_)
706 var wrapped_input_pos uint32 = wrapPosition(s.input_pos_)
707 s.last_processed_pos_ = s.input_pos_
708 return wrapped_input_pos < wrapped_last_processed_pos
709}
710
711func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
712 var last_command *command = &s.commands[len(s.commands)-1]
713 var data []byte = s.ringbuffer_.buffer_
714 var mask uint32 = s.ringbuffer_.mask_
715 var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
716 var last_copy_len uint64 = uint64(last_command.copy_len_) & 0x1FFFFFF
717 var last_processed_pos uint64 = s.last_processed_pos_ - last_copy_len
718 var max_distance uint64
719 if last_processed_pos < max_backward_distance {
720 max_distance = last_processed_pos
721 } else {
722 max_distance = max_backward_distance
723 }
724 var cmd_dist uint64 = uint64(s.dist_cache_[0])
725 var distance_code uint32 = commandRestoreDistanceCode(last_command, &s.params.dist)
726 if distance_code < numDistanceShortCodes || uint64(distance_code-(numDistanceShortCodes-1)) == cmd_dist {
727 if cmd_dist <= max_distance {
728 for *bytes != 0 && data[*wrapped_last_processed_pos&mask] == data[(uint64(*wrapped_last_processed_pos)-cmd_dist)&uint64(mask)] {
729 last_command.copy_len_++
730 (*bytes)--
731 (*wrapped_last_processed_pos)++
732 }
733 }
734
735 /* The copy length is at most the metablock size, and thus expressible. */
736 getLengthCode(uint(last_command.insert_len_), uint(int(last_command.copy_len_&0x1FFFFFF)+int(last_command.copy_len_>>25)), (last_command.dist_prefix_&0x3FF == 0), &last_command.cmd_prefix_)
737 }
738}
739
740/*
741 Processes the accumulated input data and writes
742 the new output meta-block to s.dest, if one has been
743 created (otherwise the processed input data is buffered internally).
744 If |is_last| or |force_flush| is true, an output meta-block is
745 always created. However, until |is_last| is true encoder may retain up
746 to 7 bits of the last byte of output. To force encoder to dump the remaining
747 bits use WriteMetadata() to append an empty meta-data block.
748 Returns false if the size of the input data is larger than
749 input_block_size().
750*/
751func encodeData(s *Writer, is_last bool, force_flush bool) bool {
752 var delta uint64 = unprocessedInputSize(s)
753 var bytes uint32 = uint32(delta)
754 var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_)
755 var data []byte
756 var mask uint32
757 var literal_context_mode int
758
759 data = s.ringbuffer_.buffer_
760 mask = s.ringbuffer_.mask_
761
762 /* Adding more blocks after "last" block is forbidden. */
763 if s.is_last_block_emitted_ {
764 return false
765 }
766 if is_last {
767 s.is_last_block_emitted_ = true
768 }
769
770 if delta > uint64(inputBlockSize(s)) {
771 return false
772 }
773
774 if s.params.quality == fastTwoPassCompressionQuality {
775 if s.command_buf_ == nil || cap(s.command_buf_) < int(kCompressFragmentTwoPassBlockSize) {
776 s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize)
777 s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize)
778 } else {
779 s.command_buf_ = s.command_buf_[:kCompressFragmentTwoPassBlockSize]
780 s.literal_buf_ = s.literal_buf_[:kCompressFragmentTwoPassBlockSize]
781 }
782 }
783
784 if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
785 var storage []byte
786 var storage_ix uint = uint(s.last_bytes_bits_)
787 var table_size uint
788 var table []int
789
790 if delta == 0 && !is_last {
791 /* We have no new input data and we don't have to finish the stream, so
792 nothing to do. */
793 return true
794 }
795
796 storage = s.getStorage(int(2*bytes + 503))
797 storage[0] = byte(s.last_bytes_)
798 storage[1] = byte(s.last_bytes_ >> 8)
799 table = getHashTable(s, s.params.quality, uint(bytes), &table_size)
800 if s.params.quality == fastOnePassCompressionQuality {
801 compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
802 } else {
803 compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage)
804 }
805
806 s.last_bytes_ = uint16(storage[storage_ix>>3])
807 s.last_bytes_bits_ = byte(storage_ix & 7)
808 updateLastProcessedPos(s)
809 s.writeOutput(storage[:storage_ix>>3])
810 return true
811 }
812 {
813 /* Theoretical max number of commands is 1 per 2 bytes. */
814 newsize := len(s.commands) + int(bytes)/2 + 1
815 if newsize > cap(s.commands) {
816 /* Reserve a bit more memory to allow merging with a next block
817 without reallocation: that would impact speed. */
818 newsize += int(bytes/4) + 16
819
820 new_commands := make([]command, len(s.commands), newsize)
821 if s.commands != nil {
822 copy(new_commands, s.commands)
823 }
824
825 s.commands = new_commands
826 }
827 }
828
829 initOrStitchToPreviousBlock(&s.hasher_, data, uint(mask), &s.params, uint(wrapped_last_processed_pos), uint(bytes), is_last)
830
831 literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
832
833 if len(s.commands) != 0 && s.last_insert_len_ == 0 {
834 extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
835 }
836
837 if s.params.quality == zopflificationQuality {
838 assert(s.params.hasher.type_ == 10)
839 createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
840 } else if s.params.quality == hqZopflificationQuality {
841 assert(s.params.hasher.type_ == 10)
842 createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
843 } else {
844 createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
845 }
846 {
847 var max_length uint = maxMetablockSize(&s.params)
848 var max_literals uint = max_length / 8
849 max_commands := int(max_length / 8)
850 var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
851 var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
852 var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols)
853 /* If maximal possible additional block doesn't fit metablock, flush now. */
854 /* TODO: Postpone decision until next block arrives? */
855
856 /* If block splitting is not used, then flush as soon as there is some
857 amount of commands / literals produced. */
858 if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands {
859 /* Merge with next input block. Everything will happen later. */
860 if updateLastProcessedPos(s) {
861 hasherReset(s.hasher_)
862 }
863
864 return true
865 }
866 }
867
868 /* Create the last insert-only command. */
869 if s.last_insert_len_ > 0 {
870 s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_))
871 s.num_literals_ += s.last_insert_len_
872 s.last_insert_len_ = 0
873 }
874
875 if !is_last && s.input_pos_ == s.last_flush_pos_ {
876 /* We have no new input data and we don't have to finish the stream, so
877 nothing to do. */
878 return true
879 }
880
881 assert(s.input_pos_ >= s.last_flush_pos_)
882 assert(s.input_pos_ > s.last_flush_pos_ || is_last)
883 assert(s.input_pos_-s.last_flush_pos_ <= 1<<24)
884 {
885 var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_)
886 var storage []byte = s.getStorage(int(2*metablock_size + 503))
887 var storage_ix uint = uint(s.last_bytes_bits_)
888 storage[0] = byte(s.last_bytes_)
889 storage[1] = byte(s.last_bytes_ >> 8)
890 writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
891 s.last_bytes_ = uint16(storage[storage_ix>>3])
892 s.last_bytes_bits_ = byte(storage_ix & 7)
893 s.last_flush_pos_ = s.input_pos_
894 if updateLastProcessedPos(s) {
895 hasherReset(s.hasher_)
896 }
897
898 if s.last_flush_pos_ > 0 {
899 s.prev_byte_ = data[(uint32(s.last_flush_pos_)-1)&mask]
900 }
901
902 if s.last_flush_pos_ > 1 {
903 s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
904 }
905
906 s.commands = s.commands[:0]
907 s.num_literals_ = 0
908
909 /* Save the state of the distance cache in case we need to restore it for
910 emitting an uncompressed block. */
911 copy(s.saved_dist_cache_[:], s.dist_cache_[:])
912
913 s.writeOutput(storage[:storage_ix>>3])
914 return true
915 }
916}
917
918/* Dumps remaining output bits and metadata header to |header|.
919 Returns number of produced bytes.
920 REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
921 REQUIRED: |block_size| <= (1 << 24). */
922func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint {
923 storage_ix := uint(s.last_bytes_bits_)
924 header[0] = byte(s.last_bytes_)
925 header[1] = byte(s.last_bytes_ >> 8)
926 s.last_bytes_ = 0
927 s.last_bytes_bits_ = 0
928
929 writeBits(1, 0, &storage_ix, header)
930 writeBits(2, 3, &storage_ix, header)
931 writeBits(1, 0, &storage_ix, header)
932 if block_size == 0 {
933 writeBits(2, 0, &storage_ix, header)
934 } else {
935 var nbits uint32
936 if block_size == 1 {
937 nbits = 0
938 } else {
939 nbits = log2FloorNonZero(uint(uint32(block_size)-1)) + 1
940 }
941 var nbytes uint32 = (nbits + 7) / 8
942 writeBits(2, uint64(nbytes), &storage_ix, header)
943 writeBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header)
944 }
945
946 return (storage_ix + 7) >> 3
947}
948
949func injectBytePaddingBlock(s *Writer) {
950 var seal uint32 = uint32(s.last_bytes_)
951 var seal_bits uint = uint(s.last_bytes_bits_)
952 s.last_bytes_ = 0
953 s.last_bytes_bits_ = 0
954
955 /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
956 seal |= 0x6 << seal_bits
957
958 seal_bits += 6
959
960 destination := s.tiny_buf_.u8[:]
961
962 destination[0] = byte(seal)
963 if seal_bits > 8 {
964 destination[1] = byte(seal >> 8)
965 }
966 if seal_bits > 16 {
967 destination[2] = byte(seal >> 16)
968 }
969 s.writeOutput(destination[:(seal_bits+7)>>3])
970}
971
972func checkFlushComplete(s *Writer) {
973 if s.stream_state_ == streamFlushRequested && s.err == nil {
974 s.stream_state_ = streamProcessing
975 }
976}
977
978func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte) bool {
979 var block_size_limit uint = uint(1) << s.params.lgwin
980 var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit))
981 var command_buf []uint32 = nil
982 var literal_buf []byte = nil
983 if s.params.quality != fastOnePassCompressionQuality && s.params.quality != fastTwoPassCompressionQuality {
984 return false
985 }
986
987 if s.params.quality == fastTwoPassCompressionQuality {
988 if s.command_buf_ == nil || cap(s.command_buf_) < int(buf_size) {
989 s.command_buf_ = make([]uint32, buf_size)
990 s.literal_buf_ = make([]byte, buf_size)
991 } else {
992 s.command_buf_ = s.command_buf_[:buf_size]
993 s.literal_buf_ = s.literal_buf_[:buf_size]
994 }
995
996 command_buf = s.command_buf_
997 literal_buf = s.literal_buf_
998 }
999
1000 for {
1001 if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1002 injectBytePaddingBlock(s)
1003 continue
1004 }
1005
1006 /* Compress block only when stream is not
1007 finished, there is no pending flush request, and there is either
1008 additional input or pending operation. */
1009 if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) {
1010 var block_size uint = brotli_min_size_t(block_size_limit, *available_in)
1011 var is_last bool = (*available_in == block_size) && (op == int(operationFinish))
1012 var force_flush bool = (*available_in == block_size) && (op == int(operationFlush))
1013 var max_out_size uint = 2*block_size + 503
1014 var storage []byte = nil
1015 var storage_ix uint = uint(s.last_bytes_bits_)
1016 var table_size uint
1017 var table []int
1018
1019 if force_flush && block_size == 0 {
1020 s.stream_state_ = streamFlushRequested
1021 continue
1022 }
1023
1024 storage = s.getStorage(int(max_out_size))
1025
1026 storage[0] = byte(s.last_bytes_)
1027 storage[1] = byte(s.last_bytes_ >> 8)
1028 table = getHashTable(s, s.params.quality, block_size, &table_size)
1029
1030 if s.params.quality == fastOnePassCompressionQuality {
1031 compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
1032 } else {
1033 compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage)
1034 }
1035
1036 *next_in = (*next_in)[block_size:]
1037 *available_in -= block_size
1038 var out_bytes uint = storage_ix >> 3
1039 s.writeOutput(storage[:out_bytes])
1040
1041 s.last_bytes_ = uint16(storage[storage_ix>>3])
1042 s.last_bytes_bits_ = byte(storage_ix & 7)
1043
1044 if force_flush {
1045 s.stream_state_ = streamFlushRequested
1046 }
1047 if is_last {
1048 s.stream_state_ = streamFinished
1049 }
1050 continue
1051 }
1052
1053 break
1054 }
1055
1056 checkFlushComplete(s)
1057 return true
1058}
1059
1060func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool {
1061 if *available_in > 1<<24 {
1062 return false
1063 }
1064
1065 /* Switch to metadata block workflow, if required. */
1066 if s.stream_state_ == streamProcessing {
1067 s.remaining_metadata_bytes_ = uint32(*available_in)
1068 s.stream_state_ = streamMetadataHead
1069 }
1070
1071 if s.stream_state_ != streamMetadataHead && s.stream_state_ != streamMetadataBody {
1072 return false
1073 }
1074
1075 for {
1076 if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1077 injectBytePaddingBlock(s)
1078 continue
1079 }
1080
1081 if s.input_pos_ != s.last_flush_pos_ {
1082 var result bool = encodeData(s, false, true)
1083 if !result {
1084 return false
1085 }
1086 continue
1087 }
1088
1089 if s.stream_state_ == streamMetadataHead {
1090 n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:])
1091 s.writeOutput(s.tiny_buf_.u8[:n])
1092 s.stream_state_ = streamMetadataBody
1093 continue
1094 } else {
1095 /* Exit workflow only when there is no more input and no more output.
1096 Otherwise client may continue producing empty metadata blocks. */
1097 if s.remaining_metadata_bytes_ == 0 {
1098 s.remaining_metadata_bytes_ = math.MaxUint32
1099 s.stream_state_ = streamProcessing
1100 break
1101 }
1102
1103 /* This guarantees progress in "TakeOutput" workflow. */
1104 var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16)
1105 copy(s.tiny_buf_.u8[:], (*next_in)[:c])
1106 *next_in = (*next_in)[c:]
1107 *available_in -= uint(c)
1108 s.remaining_metadata_bytes_ -= c
1109 s.writeOutput(s.tiny_buf_.u8[:c])
1110
1111 continue
1112 }
1113 }
1114
1115 return true
1116}
1117
1118func updateSizeHint(s *Writer, available_in uint) {
1119 if s.params.size_hint == 0 {
1120 var delta uint64 = unprocessedInputSize(s)
1121 var tail uint64 = uint64(available_in)
1122 var limit uint32 = 1 << 30
1123 var total uint32
1124 if (delta >= uint64(limit)) || (tail >= uint64(limit)) || ((delta + tail) >= uint64(limit)) {
1125 total = limit
1126 } else {
1127 total = uint32(delta + tail)
1128 }
1129
1130 s.params.size_hint = uint(total)
1131 }
1132}
1133
1134func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byte) bool {
1135 if !ensureInitialized(s) {
1136 return false
1137 }
1138
1139 /* Unfinished metadata block; check requirements. */
1140 if s.remaining_metadata_bytes_ != math.MaxUint32 {
1141 if uint32(*available_in) != s.remaining_metadata_bytes_ {
1142 return false
1143 }
1144 if op != int(operationEmitMetadata) {
1145 return false
1146 }
1147 }
1148
1149 if op == int(operationEmitMetadata) {
1150 updateSizeHint(s, 0) /* First data metablock might be emitted here. */
1151 return processMetadata(s, available_in, next_in)
1152 }
1153
1154 if s.stream_state_ == streamMetadataHead || s.stream_state_ == streamMetadataBody {
1155 return false
1156 }
1157
1158 if s.stream_state_ != streamProcessing && *available_in != 0 {
1159 return false
1160 }
1161
1162 if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
1163 return encoderCompressStreamFast(s, op, available_in, next_in)
1164 }
1165
1166 for {
1167 var remaining_block_size uint = remainingInputBlockSize(s)
1168
1169 if remaining_block_size != 0 && *available_in != 0 {
1170 var copy_input_size uint = brotli_min_size_t(remaining_block_size, *available_in)
1171 copyInputToRingBuffer(s, copy_input_size, *next_in)
1172 *next_in = (*next_in)[copy_input_size:]
1173 *available_in -= copy_input_size
1174 continue
1175 }
1176
1177 if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1178 injectBytePaddingBlock(s)
1179 continue
1180 }
1181
1182 /* Compress data only when stream is not
1183 finished and there is no pending flush request. */
1184 if s.stream_state_ == streamProcessing {
1185 if remaining_block_size == 0 || op != int(operationProcess) {
1186 var is_last bool = ((*available_in == 0) && op == int(operationFinish))
1187 var force_flush bool = ((*available_in == 0) && op == int(operationFlush))
1188 var result bool
1189 updateSizeHint(s, *available_in)
1190 result = encodeData(s, is_last, force_flush)
1191 if !result {
1192 return false
1193 }
1194 if force_flush {
1195 s.stream_state_ = streamFlushRequested
1196 }
1197 if is_last {
1198 s.stream_state_ = streamFinished
1199 }
1200 continue
1201 }
1202 }
1203
1204 break
1205 }
1206
1207 checkFlushComplete(s)
1208 return true
1209}
1210
1211func (w *Writer) writeOutput(data []byte) {
1212 if w.err != nil {
1213 return
1214 }
1215
1216 _, w.err = w.dst.Write(data)
1217 if w.err == nil {
1218 checkFlushComplete(w)
1219 }
1220}
Note: See TracBrowser for help on using the repository browser.