Implement serialised binary format

This commit is contained in:
RunasSudo 2021-09-02 17:17:45 +10:00
parent 31cdf3d99d
commit e9e1c63c9c
No known key found for this signature in database
GPG Key ID: 7234E476BF21C61A
10 changed files with 289 additions and 23 deletions

129
Cargo.lock generated
View File

@ -1,11 +1,24 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
@ -80,6 +93,27 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
[[package]]
name = "bytecheck"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb738a1e65989ecdcd5bba16079641bd7209688fa546e1064832fd6e012fd32a"
dependencies = [
"bytecheck_derive",
"ptr_meta",
]
[[package]]
name = "bytecheck_derive"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3b4dff26fdc9f847dab475c9fec16f2cba82d5aa1f09981b87c44520721e10a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "byteorder"
version = "1.4.3"
@ -329,6 +363,17 @@ dependencies = [
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [
"cfg-if 1.0.0",
"libc",
"wasi",
]
[[package]]
name = "git-version"
version = "0.3.4"
@ -367,6 +412,15 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash",
]
[[package]]
name = "heck"
version = "0.3.2"
@ -402,7 +456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
dependencies = [
"autocfg",
"hashbrown",
"hashbrown 0.9.1",
]
[[package]]
@ -545,6 +599,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
[[package]]
name = "opaque-debug"
version = "0.3.0"
@ -572,6 +632,7 @@ dependencies = [
"num-traits",
"paste",
"predicates",
"rkyv",
"rug",
"sha2",
"utf8-chars",
@ -668,6 +729,26 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "ptr_meta"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
dependencies = [
"ptr_meta_derive",
]
[[package]]
name = "ptr_meta_derive"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.9"
@ -724,6 +805,40 @@ version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "rend"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d0351a2e529ee30d571ef31faa5a4e0b9addaad087697b77efb20d2809e41c7"
dependencies = [
"bytecheck",
]
[[package]]
name = "rkyv"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e804c561b577f5836dc8a1962b7f7a03eae36f716dcd5f779c5d52a0e9c09a7"
dependencies = [
"bytecheck",
"hashbrown 0.11.2",
"ptr_meta",
"rend",
"rkyv_derive",
"seahash",
]
[[package]]
name = "rkyv_derive"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0afbc272334d4a4896e382508531f941a7d9505057d7424bcbed653682ce661e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "rug"
version = "1.12.0"
@ -750,6 +865,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "semver"
version = "0.11.0"
@ -876,6 +997,12 @@ dependencies = [
"libc",
]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.74"

View File

@ -32,20 +32,21 @@ paste = "1.0.5"
assert_cmd = "1.0.5"
csv = "1.1.6"
flate2 = "1.0"
rkyv = "0.7.15"
utf8-chars = "1.0.2"
xmltree = "0.10.3"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap]
#version = "3.0.0-beta.4" # proc-macro2 version conflict with rkyv
git = "https://github.com/clap-rs/clap"
branch = "master"
default-features = false
features = ["std", "derive"]
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.rug]
version = "1.12"
default-features = false
features = ["integer", "rational", "float"]
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap]
#version = "3.0.0-beta.2" # Bug 2279
git = "https://github.com/clap-rs/clap"
branch = "master"
default-features = false
features = ["std", "derive"]
[profile.test]
opt-level = 3

View File

@ -37,11 +37,11 @@ pub struct SubcmdOptions {
outfile: String,
/// Format of input file
#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")]
#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")]
r#in: Option<String>,
/// Format of output file
#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")]
#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")]
out: Option<String>,
/// Number of seats
@ -53,7 +53,9 @@ pub struct SubcmdOptions {
pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
// Auto-detect input/output formats
if cmd_opts.r#in == None {
if cmd_opts.infile.ends_with(".blt") {
if cmd_opts.infile.ends_with(".bin") {
cmd_opts.r#in = Some("bin".to_string());
} else if cmd_opts.infile.ends_with(".blt") {
cmd_opts.r#in = Some("blt".to_string());
} else if cmd_opts.infile.ends_with(".csp") {
cmd_opts.r#in = Some("csp".to_string());
@ -63,7 +65,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
}
}
if cmd_opts.out == None {
if cmd_opts.outfile.ends_with(".blt") {
if cmd_opts.outfile.ends_with(".bin") {
cmd_opts.out = Some("bin".to_string());
} else if cmd_opts.outfile.ends_with(".blt") {
cmd_opts.out = Some("blt".to_string());
} else if cmd_opts.outfile.ends_with(".csp") {
cmd_opts.out = Some("csp".to_string());
@ -77,6 +81,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
let mut election: Election<Rational>;
match cmd_opts.r#in.as_deref().unwrap() {
"bin" => {
election = parser::bin::parse_path(cmd_opts.infile);
}
"blt" => {
match parser::blt::parse_path(cmd_opts.infile) {
Ok(e) => {
@ -111,6 +118,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
let output = File::create(cmd_opts.outfile).expect("IO Error");
match cmd_opts.out.as_deref().unwrap() {
"bin" => {
writer::bin::write(election, output);
}
"blt" => {
writer::blt::write(election, output);
}

View File

@ -18,7 +18,7 @@
use crate::constraints::Constraints;
use crate::election::{CandidateState, CountState, Election};
use crate::numbers::{Fixed, GuardedFixed, NativeFloat64, Number, Rational};
use crate::parser::blt;
use crate::parser::{bin, blt};
use crate::stv::{self, STVOptions};
use crate::ties;
@ -37,8 +37,13 @@ pub struct SubcmdOptions {
// -- File input --
/// Path to the BLT file to be counted
#[clap(help_heading=Some("INPUT"))]
filename: String,
/// Input is in serialised binary format from "opentally convert"
#[clap(help_heading=Some("INPUT"), long)]
bin: bool,
// ----------------------
// -- Numbers settings --
@ -189,25 +194,25 @@ pub struct SubcmdOptions {
pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> {
// Read and count election according to --numbers
if cmd_opts.numbers == "rational" {
let mut election = election_from_file(&cmd_opts.filename)?;
let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
maybe_load_constraints(&mut election, &cmd_opts.constraints);
// Must specify ::<N> here and in a few other places because ndarray causes E0275 otherwise
count_election::<Rational>(election, cmd_opts)?;
} else if cmd_opts.numbers == "float64" {
let mut election = election_from_file(&cmd_opts.filename)?;
let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
maybe_load_constraints(&mut election, &cmd_opts.constraints);
count_election::<NativeFloat64>(election, cmd_opts)?;
} else if cmd_opts.numbers == "fixed" {
Fixed::set_dps(cmd_opts.decimals);
let mut election = election_from_file(&cmd_opts.filename)?;
let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
maybe_load_constraints(&mut election, &cmd_opts.constraints);
count_election::<Fixed>(election, cmd_opts)?;
} else if cmd_opts.numbers == "gfixed" {
GuardedFixed::set_dps(cmd_opts.decimals);
let mut election = election_from_file(&cmd_opts.filename)?;
let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
maybe_load_constraints(&mut election, &cmd_opts.constraints);
count_election::<GuardedFixed>(election, cmd_opts)?;
}
@ -215,7 +220,12 @@ pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> {
return Ok(());
}
fn election_from_file<N: Number>(path: &str) -> Result<Election<N>, i32> {
fn election_from_file<N: Number>(path: &str, bin: bool) -> Result<Election<N>, i32> {
if bin {
// BIN format
return Ok(bin::parse_path(path));
} else {
// BLT format
match blt::parse_path(path) {
Ok(e) => return Ok(e),
Err(err) => {
@ -224,6 +234,7 @@ fn election_from_file<N: Number>(path: &str) -> Result<Election<N>, i32> {
}
}
}
}
fn maybe_load_constraints<N: Number>(election: &mut Election<N>, constraints: &Option<String>) {
if let Some(c) = constraints {

View File

@ -22,12 +22,16 @@ use crate::stv::{ConstraintMode, STVOptions};
use itertools::Itertools;
use ndarray::{Array, Dimension, IxDyn};
#[cfg(not(target_arch = "wasm32"))]
use rkyv::{Archive, Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use std::ops;
/// Constraints for an [crate::election::Election]
#[derive(Debug)]
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct Constraints(pub Vec<Constraint>);
impl Constraints {
@ -121,6 +125,7 @@ impl Constraints {
/// A single dimension of constraint
#[derive(Debug)]
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct Constraint {
/// Name of this constraint
pub name: String,
@ -130,6 +135,7 @@ pub struct Constraint {
/// A group of candidates, of which a certain minimum and maximum must be elected
#[derive(Debug)]
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct ConstrainedGroup {
/// Name of this group
pub name: String,

View File

@ -21,10 +21,14 @@ use crate::numbers::Number;
use crate::sharandom::SHARandom;
use crate::stv::{QuotaMode, STVOptions};
#[cfg(not(target_arch = "wasm32"))]
use rkyv::{Archive, Archived, Deserialize, Fallible, Resolver, Serialize, with::{ArchiveWith, DeserializeWith, SerializeWith}};
use std::cmp::max;
use std::collections::HashMap;
/// An election to be counted
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct Election<N> {
/// Name of the election
pub name: String,
@ -62,6 +66,7 @@ impl<N: Number> Election<N> {
/// A candidate in an [Election]
#[derive(PartialEq, Eq, Hash)]
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct Candidate {
/// Name of the candidate
pub name: String,
@ -363,13 +368,43 @@ pub struct Vote<'a, N> {
}
/// A record of a voter's preferences
#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
pub struct Ballot<N> {
/// Original value/weight of the ballot
#[cfg_attr(not(target_arch = "wasm32"), with(SerializedNum))]
pub orig_value: N,
/// Indexes of candidates preferenced on the ballot
pub preferences: Vec<usize>,
}
/// rkyv-serialized representation of [Number]
#[cfg(not(target_arch = "wasm32"))]
pub struct SerializedNum;
#[cfg(not(target_arch = "wasm32"))]
impl<N: Number> ArchiveWith<N> for SerializedNum {
type Archived = Archived<String>;
type Resolver = Resolver<String>;
unsafe fn resolve_with(field: &N, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) {
field.to_string().resolve(pos, resolver, out);
}
}
#[cfg(not(target_arch = "wasm32"))]
impl<N: Number, S: Fallible + ?Sized> SerializeWith<N, S> for SerializedNum where String: Serialize<S> {
fn serialize_with(field: &N, serializer: &mut S) -> Result<Self::Resolver, S::Error> {
return field.to_string().serialize(serializer);
}
}
#[cfg(not(target_arch = "wasm32"))]
impl<N: Number, D: Fallible + ?Sized> DeserializeWith<Archived<String>, N, D> for SerializedNum where Archived<String>: Deserialize<String, D> {
fn deserialize_with(field: &Archived<String>, deserializer: &mut D) -> Result<N, D::Error> {
return Ok(N::parse(&field.deserialize(deserializer)?));
}
}
/// State of a [Candidate] during a count
#[allow(dead_code)]
#[derive(PartialEq)]

33
src/parser/bin.rs Normal file
View File

@ -0,0 +1,33 @@
/* OpenTally: Open-source election vote counting
* Copyright © 2021 Lee Yingtong Li (RunasSudo)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use crate::election::Election;
use crate::numbers::Number;
use rkyv::{Deserialize, Infallible, archived_root};
use std::fs;
use std::path::Path;
/// Parse the given BIN file
pub fn parse_path<P: AsRef<Path>, N: Number>(path: P) -> Election<N> {
let content = fs::read(path).expect("IO Error");
let archived = unsafe {
archived_root::<Election<N>>(&content)
};
return archived.deserialize(&mut Infallible).unwrap();
}

View File

@ -15,6 +15,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/// BIN file parser
#[cfg(not(target_arch = "wasm32"))]
pub mod bin;
/// BLT file parser
pub mod blt;
/// CSP file parser

35
src/writer/bin.rs Normal file
View File

@ -0,0 +1,35 @@
/* OpenTally: Open-source election vote counting
* Copyright © 2021 Lee Yingtong Li (RunasSudo)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use crate::election::Election;
use crate::numbers::Number;
use rkyv::ser::{Serializer, serializers::AllocSerializer};
use std::io::{BufWriter, Write};
/// Write the [Election] into BIN format
pub fn write<W: Write, N: Number>(election: Election<N>, output: W) {
// Serialize data using rkyv
let mut serializer = AllocSerializer::<256>::default();
serializer.serialize_value(&election).unwrap();
let buffer = serializer.into_serializer().into_inner();
// Write output
let mut output = BufWriter::new(output);
output.write(&buffer).expect("IO Error");
}

View File

@ -15,6 +15,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/// BIN file writer
#[cfg(not(target_arch = "wasm32"))]
pub mod bin;
/// BLT file writer
pub mod blt;
/// CSP file writer