serialize.go (84 lines of code) (raw):

// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved package qf import ( "encoding/binary" "fmt" "io" "os" "unsafe" ) // qfVersion is a version number for the // on disk representation format. Any time incompatible // changes are made, it is bumped const qfVersion = uint64(0x0004) // QFHeader describes a serialized quotient filter type QFHeader struct { // a version number which changes as the storage representation // changes Version uint64 // then number of entries in the stored quotient filter Entries uint64 // the number of bits allocated to the quotient filter. the // length of the hash vector on disk will then be 1 << QBits QBits uint64 // the number of bits per bucket of storage represented in the // quotient filter. May be zero if no external storage is in // use StorageBits uint64 // whether the quotient filters use bitpacked storage BitPacked bool } // ReadHeaderFromPath reads and returns the header from a serialized quotient filter // at a specified path func ReadHeaderFromPath(path string) (*QFHeader, error) { stream, err := os.Open(path) if err != nil { return nil, err } defer stream.Close() var h QFHeader if err = binary.Read(stream, binary.LittleEndian, &h); err != nil { return nil, err } return &h, nil } // WriteTo allows the quotient filter to be written to a stream // // WARNING: the default storage format is very fast, but not portable // to architectures of differing word length or endianness func (qf *Filter) WriteTo(stream io.Writer) (i int64, err error) { h := QFHeader{ Version: qfVersion, Entries: qf.entries, QBits: uint64(qf.qBits), StorageBits: uint64(qf.config.BitsOfStoragePerEntry), BitPacked: qf.config.BitPacked, } if err = binary.Write(stream, binary.LittleEndian, h); err != nil { return } i += int64(unsafe.Sizeof(h)) x, err := qf.filter.WriteTo(stream) i += x if err != nil { return } if qf.storage != nil { x, err = qf.storage.WriteTo(stream) i += x if err != nil { return } } return } // ReadFrom allows the quotient filter to be read from a stream // // WARNING: the default storage format is very fast, but not portable // to architectures of differing word length or endianness func (qf *Filter) ReadFrom(stream io.Reader) (i int64, err error) { var h QFHeader if err = binary.Read(stream, binary.LittleEndian, &h); err != nil { return } i += int64(unsafe.Sizeof(h)) if h.Version != qfVersion { return i, fmt.Errorf("incompatible file format: version is %d, expected %d", h.Version, qfVersion) } qf.entries = h.Entries qf.initForQuotientBits(uint(h.QBits)) n, err := qf.filter.ReadFrom(stream) i += n if err != nil { return } // read bits if h.StorageBits > 0 { qf.config.BitsOfStoragePerEntry = uint(h.StorageBits) if qf.storage == nil { qf.storage = qf.allocfn(0, 0) } n, err = qf.storage.ReadFrom(stream) i += n if err != nil { return } } return }