Browse Source
* update macaron to v1.5 of fork * update macaron to v1.5 of fork * test gzip PR * add push method impl to context_tests * use proper gzip commit Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>tags/v1.13.0-rc1
techknowlogick
5 years ago
committed by
GitHub
53 changed files with 2595 additions and 638 deletions
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,104 @@ |
|||||||
|
package zstd |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"encoding/binary" |
||||||
|
"errors" |
||||||
|
"fmt" |
||||||
|
"io" |
||||||
|
|
||||||
|
"github.com/klauspost/compress/huff0" |
||||||
|
) |
||||||
|
|
||||||
|
type dict struct { |
||||||
|
id uint32 |
||||||
|
|
||||||
|
litDec *huff0.Scratch |
||||||
|
llDec, ofDec, mlDec sequenceDec |
||||||
|
offsets [3]int |
||||||
|
content []byte |
||||||
|
} |
||||||
|
|
||||||
|
var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} |
||||||
|
|
||||||
|
// Load a dictionary as described in
|
||||||
|
// https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
|
||||||
|
func loadDict(b []byte) (*dict, error) { |
||||||
|
// Check static field size.
|
||||||
|
if len(b) <= 8+(3*4) { |
||||||
|
return nil, io.ErrUnexpectedEOF |
||||||
|
} |
||||||
|
d := dict{ |
||||||
|
llDec: sequenceDec{fse: &fseDecoder{}}, |
||||||
|
ofDec: sequenceDec{fse: &fseDecoder{}}, |
||||||
|
mlDec: sequenceDec{fse: &fseDecoder{}}, |
||||||
|
} |
||||||
|
if !bytes.Equal(b[:4], dictMagic[:]) { |
||||||
|
return nil, ErrMagicMismatch |
||||||
|
} |
||||||
|
d.id = binary.LittleEndian.Uint32(b[4:8]) |
||||||
|
if d.id == 0 { |
||||||
|
return nil, errors.New("dictionaries cannot have ID 0") |
||||||
|
} |
||||||
|
|
||||||
|
// Read literal table
|
||||||
|
var err error |
||||||
|
d.litDec, b, err = huff0.ReadTable(b[8:], nil) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
br := byteReader{ |
||||||
|
b: b, |
||||||
|
off: 0, |
||||||
|
} |
||||||
|
readDec := func(i tableIndex, dec *fseDecoder) error { |
||||||
|
if err := dec.readNCount(&br, uint16(maxTableSymbol[i])); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
if br.overread() { |
||||||
|
return io.ErrUnexpectedEOF |
||||||
|
} |
||||||
|
err = dec.transform(symbolTableX[i]) |
||||||
|
if err != nil { |
||||||
|
println("Transform table error:", err) |
||||||
|
return err |
||||||
|
} |
||||||
|
if debug { |
||||||
|
println("Read table ok", "symbolLen:", dec.symbolLen) |
||||||
|
} |
||||||
|
// Set decoders as predefined so they aren't reused.
|
||||||
|
dec.preDefined = true |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
if err := readDec(tableOffsets, d.ofDec.fse); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
if err := readDec(tableMatchLengths, d.mlDec.fse); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
if err := readDec(tableLiteralLengths, d.llDec.fse); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
if br.remain() < 12 { |
||||||
|
return nil, io.ErrUnexpectedEOF |
||||||
|
} |
||||||
|
|
||||||
|
d.offsets[0] = int(br.Uint32()) |
||||||
|
br.advance(4) |
||||||
|
d.offsets[1] = int(br.Uint32()) |
||||||
|
br.advance(4) |
||||||
|
d.offsets[2] = int(br.Uint32()) |
||||||
|
br.advance(4) |
||||||
|
if d.offsets[0] <= 0 || d.offsets[1] <= 0 || d.offsets[2] <= 0 { |
||||||
|
return nil, errors.New("invalid offset in dictionary") |
||||||
|
} |
||||||
|
d.content = make([]byte, br.remain()) |
||||||
|
copy(d.content, br.unread()) |
||||||
|
if d.offsets[0] > len(d.content) || d.offsets[1] > len(d.content) || d.offsets[2] > len(d.content) { |
||||||
|
return nil, fmt.Errorf("initial offset bigger than dictionary content size %d, offsets: %v", len(d.content), d.offsets) |
||||||
|
} |
||||||
|
|
||||||
|
return &d, nil |
||||||
|
} |
||||||
@ -0,0 +1,518 @@ |
|||||||
|
// Copyright 2019+ Klaus Post. All rights reserved.
|
||||||
|
// License information can be found in the LICENSE file.
|
||||||
|
// Based on work by Yann Collet, released under BSD License.
|
||||||
|
|
||||||
|
package zstd |
||||||
|
|
||||||
|
import "fmt" |
||||||
|
|
||||||
|
const ( |
||||||
|
betterLongTableBits = 19 // Bits used in the long match table
|
||||||
|
betterLongTableSize = 1 << betterLongTableBits // Size of the table
|
||||||
|
|
||||||
|
// Note: Increasing the short table bits or making the hash shorter
|
||||||
|
// can actually lead to compression degradation since it will 'steal' more from the
|
||||||
|
// long match table and match offsets are quite big.
|
||||||
|
// This greatly depends on the type of input.
|
||||||
|
betterShortTableBits = 13 // Bits used in the short match table
|
||||||
|
betterShortTableSize = 1 << betterShortTableBits // Size of the table
|
||||||
|
) |
||||||
|
|
||||||
|
type prevEntry struct { |
||||||
|
offset int32 |
||||||
|
prev int32 |
||||||
|
} |
||||||
|
|
||||||
|
// betterFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches.
|
||||||
|
// The long match table contains the previous entry with the same hash,
|
||||||
|
// effectively making it a "chain" of length 2.
|
||||||
|
// When we find a long match we choose between the two values and select the longest.
|
||||||
|
// When we find a short match, after checking the long, we check if we can find a long at n+1
|
||||||
|
// and that it is longer (lazy matching).
|
||||||
|
type betterFastEncoder struct { |
||||||
|
fastBase |
||||||
|
table [betterShortTableSize]tableEntry |
||||||
|
longTable [betterLongTableSize]prevEntry |
||||||
|
} |
||||||
|
|
||||||
|
// Encode improves compression...
|
||||||
|
func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { |
||||||
|
const ( |
||||||
|
// Input margin is the number of bytes we read (8)
|
||||||
|
// and the maximum we will read ahead (2)
|
||||||
|
inputMargin = 8 + 2 |
||||||
|
minNonLiteralBlockSize = 16 |
||||||
|
) |
||||||
|
|
||||||
|
// Protect against e.cur wraparound.
|
||||||
|
for e.cur >= bufferReset { |
||||||
|
if len(e.hist) == 0 { |
||||||
|
for i := range e.table[:] { |
||||||
|
e.table[i] = tableEntry{} |
||||||
|
} |
||||||
|
for i := range e.longTable[:] { |
||||||
|
e.longTable[i] = prevEntry{} |
||||||
|
} |
||||||
|
e.cur = e.maxMatchOff |
||||||
|
break |
||||||
|
} |
||||||
|
// Shift down everything in the table that isn't already too far away.
|
||||||
|
minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff |
||||||
|
for i := range e.table[:] { |
||||||
|
v := e.table[i].offset |
||||||
|
if v < minOff { |
||||||
|
v = 0 |
||||||
|
} else { |
||||||
|
v = v - e.cur + e.maxMatchOff |
||||||
|
} |
||||||
|
e.table[i].offset = v |
||||||
|
} |
||||||
|
for i := range e.longTable[:] { |
||||||
|
v := e.longTable[i].offset |
||||||
|
v2 := e.longTable[i].prev |
||||||
|
if v < minOff { |
||||||
|
v = 0 |
||||||
|
v2 = 0 |
||||||
|
} else { |
||||||
|
v = v - e.cur + e.maxMatchOff |
||||||
|
if v2 < minOff { |
||||||
|
v2 = 0 |
||||||
|
} else { |
||||||
|
v2 = v2 - e.cur + e.maxMatchOff |
||||||
|
} |
||||||
|
} |
||||||
|
e.longTable[i] = prevEntry{ |
||||||
|
offset: v, |
||||||
|
prev: v2, |
||||||
|
} |
||||||
|
} |
||||||
|
e.cur = e.maxMatchOff |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
s := e.addBlock(src) |
||||||
|
blk.size = len(src) |
||||||
|
if len(src) < minNonLiteralBlockSize { |
||||||
|
blk.extraLits = len(src) |
||||||
|
blk.literals = blk.literals[:len(src)] |
||||||
|
copy(blk.literals, src) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
// Override src
|
||||||
|
src = e.hist |
||||||
|
sLimit := int32(len(src)) - inputMargin |
||||||
|
// stepSize is the number of bytes to skip on every main loop iteration.
|
||||||
|
// It should be >= 1.
|
||||||
|
const stepSize = 1 |
||||||
|
|
||||||
|
const kSearchStrength = 9 |
||||||
|
|
||||||
|
// nextEmit is where in src the next emitLiteral should start from.
|
||||||
|
nextEmit := s |
||||||
|
cv := load6432(src, s) |
||||||
|
|
||||||
|
// Relative offsets
|
||||||
|
offset1 := int32(blk.recentOffsets[0]) |
||||||
|
offset2 := int32(blk.recentOffsets[1]) |
||||||
|
|
||||||
|
addLiterals := func(s *seq, until int32) { |
||||||
|
if until == nextEmit { |
||||||
|
return |
||||||
|
} |
||||||
|
blk.literals = append(blk.literals, src[nextEmit:until]...) |
||||||
|
s.litLen = uint32(until - nextEmit) |
||||||
|
} |
||||||
|
if debug { |
||||||
|
println("recent offsets:", blk.recentOffsets) |
||||||
|
} |
||||||
|
|
||||||
|
encodeLoop: |
||||||
|
for { |
||||||
|
var t int32 |
||||||
|
// We allow the encoder to optionally turn off repeat offsets across blocks
|
||||||
|
canRepeat := len(blk.sequences) > 2 |
||||||
|
var matched int32 |
||||||
|
|
||||||
|
for { |
||||||
|
if debugAsserts && canRepeat && offset1 == 0 { |
||||||
|
panic("offset0 was 0") |
||||||
|
} |
||||||
|
|
||||||
|
nextHashS := hash5(cv, betterShortTableBits) |
||||||
|
nextHashL := hash8(cv, betterLongTableBits) |
||||||
|
candidateL := e.longTable[nextHashL] |
||||||
|
candidateS := e.table[nextHashS] |
||||||
|
|
||||||
|
const repOff = 1 |
||||||
|
repIndex := s - offset1 + repOff |
||||||
|
off := s + e.cur |
||||||
|
e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset} |
||||||
|
e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)} |
||||||
|
|
||||||
|
if canRepeat { |
||||||
|
if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { |
||||||
|
// Consider history as well.
|
||||||
|
var seq seq |
||||||
|
lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) |
||||||
|
|
||||||
|
seq.matchLen = uint32(lenght - zstdMinMatch) |
||||||
|
|
||||||
|
// We might be able to match backwards.
|
||||||
|
// Extend as long as we can.
|
||||||
|
start := s + repOff |
||||||
|
// We end the search early, so we don't risk 0 literals
|
||||||
|
// and have to do special offset treatment.
|
||||||
|
startLimit := nextEmit + 1 |
||||||
|
|
||||||
|
tMin := s - e.maxMatchOff |
||||||
|
if tMin < 0 { |
||||||
|
tMin = 0 |
||||||
|
} |
||||||
|
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { |
||||||
|
repIndex-- |
||||||
|
start-- |
||||||
|
seq.matchLen++ |
||||||
|
} |
||||||
|
addLiterals(&seq, start) |
||||||
|
|
||||||
|
// rep 0
|
||||||
|
seq.offset = 1 |
||||||
|
if debugSequences { |
||||||
|
println("repeat sequence", seq, "next s:", s) |
||||||
|
} |
||||||
|
blk.sequences = append(blk.sequences, seq) |
||||||
|
|
||||||
|
// Index match start+1 (long) -> s - 1
|
||||||
|
index0 := s + repOff |
||||||
|
s += lenght + repOff |
||||||
|
|
||||||
|
nextEmit = s |
||||||
|
if s >= sLimit { |
||||||
|
if debug { |
||||||
|
println("repeat ended", s, lenght) |
||||||
|
|
||||||
|
} |
||||||
|
break encodeLoop |
||||||
|
} |
||||||
|
// Index skipped...
|
||||||
|
for index0 < s-1 { |
||||||
|
cv0 := load6432(src, index0) |
||||||
|
cv1 := cv0 >> 8 |
||||||
|
h0 := hash8(cv0, betterLongTableBits) |
||||||
|
off := index0 + e.cur |
||||||
|
e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} |
||||||
|
e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} |
||||||
|
index0 += 2 |
||||||
|
} |
||||||
|
cv = load6432(src, s) |
||||||
|
continue |
||||||
|
} |
||||||
|
const repOff2 = 1 |
||||||
|
|
||||||
|
// We deviate from the reference encoder and also check offset 2.
|
||||||
|
// Still slower and not much better, so disabled.
|
||||||
|
// repIndex = s - offset2 + repOff2
|
||||||
|
if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { |
||||||
|
// Consider history as well.
|
||||||
|
var seq seq |
||||||
|
lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) |
||||||
|
|
||||||
|
seq.matchLen = uint32(lenght - zstdMinMatch) |
||||||
|
|
||||||
|
// We might be able to match backwards.
|
||||||
|
// Extend as long as we can.
|
||||||
|
start := s + repOff2 |
||||||
|
// We end the search early, so we don't risk 0 literals
|
||||||
|
// and have to do special offset treatment.
|
||||||
|
startLimit := nextEmit + 1 |
||||||
|
|
||||||
|
tMin := s - e.maxMatchOff |
||||||
|
if tMin < 0 { |
||||||
|
tMin = 0 |
||||||
|
} |
||||||
|
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { |
||||||
|
repIndex-- |
||||||
|
start-- |
||||||
|
seq.matchLen++ |
||||||
|
} |
||||||
|
addLiterals(&seq, start) |
||||||
|
|
||||||
|
// rep 2
|
||||||
|
seq.offset = 2 |
||||||
|
if debugSequences { |
||||||
|
println("repeat sequence 2", seq, "next s:", s) |
||||||
|
} |
||||||
|
blk.sequences = append(blk.sequences, seq) |
||||||
|
|
||||||
|
index0 := s + repOff2 |
||||||
|
s += lenght + repOff2 |
||||||
|
nextEmit = s |
||||||
|
if s >= sLimit { |
||||||
|
if debug { |
||||||
|
println("repeat ended", s, lenght) |
||||||
|
|
||||||
|
} |
||||||
|
break encodeLoop |
||||||
|
} |
||||||
|
|
||||||
|
// Index skipped...
|
||||||
|
for index0 < s-1 { |
||||||
|
cv0 := load6432(src, index0) |
||||||
|
cv1 := cv0 >> 8 |
||||||
|
h0 := hash8(cv0, betterLongTableBits) |
||||||
|
off := index0 + e.cur |
||||||
|
e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} |
||||||
|
e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} |
||||||
|
index0 += 2 |
||||||
|
} |
||||||
|
cv = load6432(src, s) |
||||||
|
// Swap offsets
|
||||||
|
offset1, offset2 = offset2, offset1 |
||||||
|
continue |
||||||
|
} |
||||||
|
} |
||||||
|
// Find the offsets of our two matches.
|
||||||
|
coffsetL := candidateL.offset - e.cur |
||||||
|
coffsetLP := candidateL.prev - e.cur |
||||||
|
|
||||||
|
// Check if we have a long match.
|
||||||
|
if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { |
||||||
|
// Found a long match, at least 8 bytes.
|
||||||
|
matched = e.matchlen(s+8, coffsetL+8, src) + 8 |
||||||
|
t = coffsetL |
||||||
|
if debugAsserts && s <= t { |
||||||
|
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) |
||||||
|
} |
||||||
|
if debugAsserts && s-t > e.maxMatchOff { |
||||||
|
panic("s - t >e.maxMatchOff") |
||||||
|
} |
||||||
|
if debugMatches { |
||||||
|
println("long match") |
||||||
|
} |
||||||
|
|
||||||
|
if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { |
||||||
|
// Found a long match, at least 8 bytes.
|
||||||
|
prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8 |
||||||
|
if prevMatch > matched { |
||||||
|
matched = prevMatch |
||||||
|
t = coffsetLP |
||||||
|
} |
||||||
|
if debugAsserts && s <= t { |
||||||
|
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) |
||||||
|
} |
||||||
|
if debugAsserts && s-t > e.maxMatchOff { |
||||||
|
panic("s - t >e.maxMatchOff") |
||||||
|
} |
||||||
|
if debugMatches { |
||||||
|
println("long match") |
||||||
|
} |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
// Check if we have a long match on prev.
|
||||||
|
if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { |
||||||
|
// Found a long match, at least 8 bytes.
|
||||||
|
matched = e.matchlen(s+8, coffsetLP+8, src) + 8 |
||||||
|
t = coffsetLP |
||||||
|
if debugAsserts && s <= t { |
||||||
|
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) |
||||||
|
} |
||||||
|
if debugAsserts && s-t > e.maxMatchOff { |
||||||
|
panic("s - t >e.maxMatchOff") |
||||||
|
} |
||||||
|
if debugMatches { |
||||||
|
println("long match") |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
coffsetS := candidateS.offset - e.cur |
||||||
|
|
||||||
|
// Check if we have a short match.
|
||||||
|
if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { |
||||||
|
// found a regular match
|
||||||
|
matched = e.matchlen(s+4, coffsetS+4, src) + 4 |
||||||
|
|
||||||
|
// See if we can find a long match at s+1
|
||||||
|
const checkAt = 1 |
||||||
|
cv := load6432(src, s+checkAt) |
||||||
|
nextHashL = hash8(cv, betterLongTableBits) |
||||||
|
candidateL = e.longTable[nextHashL] |
||||||
|
coffsetL = candidateL.offset - e.cur |
||||||
|
|
||||||
|
// We can store it, since we have at least a 4 byte match.
|
||||||
|
e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset} |
||||||
|
if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { |
||||||
|
// Found a long match, at least 8 bytes.
|
||||||
|
matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 |
||||||
|
if matchedNext > matched { |
||||||
|
t = coffsetL |
||||||
|
s += checkAt |
||||||
|
matched = matchedNext |
||||||
|
if debugMatches { |
||||||
|
println("long match (after short)") |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Check prev long...
|
||||||
|
coffsetL = candidateL.prev - e.cur |
||||||
|
if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { |
||||||
|
// Found a long match, at least 8 bytes.
|
||||||
|
matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 |
||||||
|
if matchedNext > matched { |
||||||
|
t = coffsetL |
||||||
|
s += checkAt |
||||||
|
matched = matchedNext |
||||||
|
if debugMatches { |
||||||
|
println("prev long match (after short)") |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
t = coffsetS |
||||||
|
if debugAsserts && s <= t { |
||||||
|
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) |
||||||
|
} |
||||||
|
if debugAsserts && s-t > e.maxMatchOff { |
||||||
|
panic("s - t >e.maxMatchOff") |
||||||
|
} |
||||||
|
if debugAsserts && t < 0 { |
||||||
|
panic("t<0") |
||||||
|
} |
||||||
|
if debugMatches { |
||||||
|
println("short match") |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
// No match found, move forward in input.
|
||||||
|
s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) |
||||||
|
if s >= sLimit { |
||||||
|
break encodeLoop |
||||||
|
} |
||||||
|
cv = load6432(src, s) |
||||||
|
} |
||||||
|
|
||||||
|
// A 4-byte match has been found. Update recent offsets.
|
||||||
|
// We'll later see if more than 4 bytes.
|
||||||
|
offset2 = offset1 |
||||||
|
offset1 = s - t |
||||||
|
|
||||||
|
if debugAsserts && s <= t { |
||||||
|
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) |
||||||
|
} |
||||||
|
|
||||||
|
if debugAsserts && canRepeat && int(offset1) > len(src) { |
||||||
|
panic("invalid offset") |
||||||
|
} |
||||||
|
|
||||||
|
// Extend the n-byte match as long as possible.
|
||||||
|
l := matched |
||||||
|
|
||||||
|
// Extend backwards
|
||||||
|
tMin := s - e.maxMatchOff |
||||||
|
if tMin < 0 { |
||||||
|
tMin = 0 |
||||||
|
} |
||||||
|
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { |
||||||
|
s-- |
||||||
|
t-- |
||||||
|
l++ |
||||||
|
} |
||||||
|
|
||||||
|
// Write our sequence
|
||||||
|
var seq seq |
||||||
|
seq.litLen = uint32(s - nextEmit) |
||||||
|
seq.matchLen = uint32(l - zstdMinMatch) |
||||||
|
if seq.litLen > 0 { |
||||||
|
blk.literals = append(blk.literals, src[nextEmit:s]...) |
||||||
|
} |
||||||
|
seq.offset = uint32(s-t) + 3 |
||||||
|
s += l |
||||||
|
if debugSequences { |
||||||
|
println("sequence", seq, "next s:", s) |
||||||
|
} |
||||||
|
blk.sequences = append(blk.sequences, seq) |
||||||
|
nextEmit = s |
||||||
|
if s >= sLimit { |
||||||
|
break encodeLoop |
||||||
|
} |
||||||
|
|
||||||
|
// Index match start+1 (long) -> s - 1
|
||||||
|
index0 := s - l + 1 |
||||||
|
for index0 < s-1 { |
||||||
|
cv0 := load6432(src, index0) |
||||||
|
cv1 := cv0 >> 8 |
||||||
|
h0 := hash8(cv0, betterLongTableBits) |
||||||
|
off := index0 + e.cur |
||||||
|
e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} |
||||||
|
e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} |
||||||
|
index0 += 2 |
||||||
|
} |
||||||
|
|
||||||
|
cv = load6432(src, s) |
||||||
|
if !canRepeat { |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
// Check offset 2
|
||||||
|
for { |
||||||
|
o2 := s - offset2 |
||||||
|
if load3232(src, o2) != uint32(cv) { |
||||||
|
// Do regular search
|
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
// Store this, since we have it.
|
||||||
|
nextHashS := hash5(cv, betterShortTableBits) |
||||||
|
nextHashL := hash8(cv, betterLongTableBits) |
||||||
|
|
||||||
|
// We have at least 4 byte match.
|
||||||
|
// No need to check backwards. We come straight from a match
|
||||||
|
l := 4 + e.matchlen(s+4, o2+4, src) |
||||||
|
|
||||||
|
e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} |
||||||
|
e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)} |
||||||
|
seq.matchLen = uint32(l) - zstdMinMatch |
||||||
|
seq.litLen = 0 |
||||||
|
|
||||||
|
// Since litlen is always 0, this is offset 1.
|
||||||
|
seq.offset = 1 |
||||||
|
s += l |
||||||
|
nextEmit = s |
||||||
|
if debugSequences { |
||||||
|
println("sequence", seq, "next s:", s) |
||||||
|
} |
||||||
|
blk.sequences = append(blk.sequences, seq) |
||||||
|
|
||||||
|
// Swap offset 1 and 2.
|
||||||
|
offset1, offset2 = offset2, offset1 |
||||||
|
if s >= sLimit { |
||||||
|
// Finished
|
||||||
|
break encodeLoop |
||||||
|
} |
||||||
|
cv = load6432(src, s) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if int(nextEmit) < len(src) { |
||||||
|
blk.literals = append(blk.literals, src[nextEmit:]...) |
||||||
|
blk.extraLits = len(src) - int(nextEmit) |
||||||
|
} |
||||||
|
blk.recentOffsets[0] = uint32(offset1) |
||||||
|
blk.recentOffsets[1] = uint32(offset2) |
||||||
|
if debug { |
||||||
|
println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// EncodeNoHist will encode a block with no history and no following blocks.
|
||||||
|
// Most notable difference is that src will not be copied for history and
|
||||||
|
// we do not need to check for max match length.
|
||||||
|
func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { |
||||||
|
e.Encode(blk, src) |
||||||
|
} |
||||||
Loading…
Reference in new issue