Rename Raw-rs to Rawkit (#2088)

* Rename within files

* Rename in CI

* Rename the folder and file names

* Rename raw_rs to rawkit

* Add example to README

* Add initial documentation

* Small API changes and extra documentation

* Bump versions and stuff

* Readme improvements

* Merge proc-macro crates into one

* Add README to rawkit-proc-macros

* Remove keywords and categories

* Add licenses to rawkit-proc-macros

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

libraries/rawkit/src/decoder/arw1.rs

@@ -0,0 +1,109 @@
+use crate::tiff::file::TiffRead;
+use crate::tiff::tags::SonyDataOffset;
+use crate::tiff::Ifd;
+use crate::{RawImage, SubtractBlack, Transform};
+
+use bitstream_io::{BitRead, BitReader, Endianness, BE};
+use std::io::{Read, Seek};
+
+pub fn decode_a100<R: Read + Seek>(ifd: Ifd, file: &mut TiffRead<R>) -> RawImage {
+	let data_offset = ifd.get_value::<SonyDataOffset, _>(file).unwrap();
+
+	let image_width = 3881;
+	let image_height = 2608;
+
+	file.seek_from_start(data_offset).unwrap();
+	let mut image = sony_arw_load_raw(image_width, image_height, &mut BitReader::<_, BE>::new(file)).unwrap();
+
+	let len = image.len();
+	image[len - image_width..].fill(0);
+
+	RawImage {
+		data: image,
+		width: image_width,
+		height: image_height,
+		cfa_pattern: todo!(),
+		#[allow(unreachable_code)]
+		maximum: (1 << 12) - 1,
+		black: SubtractBlack::None,
+		transform: Transform::Horizontal,
+		camera_model: None,
+		camera_white_balance: None,
+		white_balance: None,
+		camera_to_rgb: None,
+	}
+}
+
+fn read_and_huffman_decode_file<R: Read + Seek, E: Endianness>(huff: &[u16], file: &mut BitReader<R, E>) -> u32 {
+	let number_of_bits = huff[0].into();
+	let huffman_table = &huff[1..];
+
+	// `number_of_bits` will be no more than 32, so the result is put into a u32
+	let bits: u32 = file.read(number_of_bits).unwrap();
+	let bits = bits as usize;
+
+	let bits_to_seek_from = huffman_table[bits].to_le_bytes()[1] as i64 - number_of_bits as i64;
+	file.seek_bits(std::io::SeekFrom::Current(bits_to_seek_from)).unwrap();
+
+	huffman_table[bits].to_le_bytes()[0].into()
+}
+
+fn read_n_bits_from_file<R: Read + Seek, E: Endianness>(number_of_bits: u32, file: &mut BitReader<R, E>) -> u32 {
+	// `number_of_bits` will be no more than 32, so the result is put into a u32
+	file.read(number_of_bits).unwrap()
+}
+
+/// ljpeg is a lossless variant of JPEG which gets used for decoding the embedded (thumbnail) preview images in raw files
+fn ljpeg_diff<R: Read + Seek, E: Endianness>(huff: &[u16], file: &mut BitReader<R, E>, dng_version: Option<u32>) -> i32 {
+	let length = read_and_huffman_decode_file(huff, file);
+
+	if length == 16 && dng_version.map(|x| x >= 0x1010000).unwrap_or(true) {
+		return -32768;
+	}
+
+	let diff = read_n_bits_from_file(length, file) as i32;
+
+	if length == 0 || (diff & (1 << (length - 1))) == 0 {
+		diff - (1 << length) - 1
+	} else {
+		diff
+	}
+}
+
+fn sony_arw_load_raw<R: Read + Seek>(width: usize, height: usize, file: &mut BitReader<R, BE>) -> Option<Vec<u16>> {
+	const TABLE: [u16; 18] = [
+		0x0f11, 0x0f10, 0x0e0f, 0x0d0e, 0x0c0d, 0x0b0c, 0x0a0b, 0x090a, 0x0809, 0x0708, 0x0607, 0x0506, 0x0405, 0x0304, 0x0303, 0x0300, 0x0202, 0x0201,
+	];
+
+	let mut huffman_table = [0_u16; 32770];
+	// The first element is the number of bits to read
+	huffman_table[0] = 15;
+
+	let mut n = 0;
+	for x in TABLE {
+		let first_byte = x >> 8;
+		let repeats = 0x8000 >> first_byte;
+		for _ in 0_u16..repeats {
+			n += 1;
+			huffman_table[n] = x;
+		}
+	}
+
+	let mut sum = 0;
+	let mut image = vec![0_u16; width * height];
+	for column in (0..width).rev() {
+		for row in (0..height).step_by(2).chain((1..height).step_by(2)) {
+			sum += ljpeg_diff(&huffman_table, file, None);
+
+			if (sum >> 12) != 0 {
+				return None;
+			}
+
+			if row < height {
+				image[row * width + column] = sum as u16;
+			}
+		}
+	}
+
+	Some(image)
+}

libraries/rawkit/src/decoder/arw2.rs

@@ -0,0 +1,127 @@
+use crate::tiff::file::{Endian, TiffRead};
+use crate::tiff::tags::{BitsPerSample, CfaPattern, CfaPatternDim, Compression, ImageLength, ImageWidth, SonyToneCurve, StripByteCounts, StripOffsets, Tag, WhiteBalanceRggbLevels};
+use crate::tiff::values::CurveLookupTable;
+use crate::tiff::{Ifd, TiffError};
+use crate::{RawImage, SubtractBlack, Transform};
+
+use rawkit_proc_macros::Tag;
+use std::io::{Read, Seek};
+
+#[allow(dead_code)]
+#[derive(Tag)]
+struct Arw2Ifd {
+	image_width: ImageWidth,
+	image_height: ImageLength,
+	bits_per_sample: BitsPerSample,
+	compression: Compression,
+	cfa_pattern: CfaPattern,
+	cfa_pattern_dim: CfaPatternDim,
+	strip_offsets: StripOffsets,
+	strip_byte_counts: StripByteCounts,
+	sony_tone_curve: SonyToneCurve,
+	white_balance_levels: Option<WhiteBalanceRggbLevels>,
+}
+
+pub fn decode<R: Read + Seek>(ifd: Ifd, file: &mut TiffRead<R>) -> RawImage {
+	let ifd = ifd.get_value::<Arw2Ifd, _>(file).unwrap();
+
+	assert!(ifd.strip_offsets.len() == ifd.strip_byte_counts.len());
+	assert!(ifd.strip_offsets.len() == 1);
+	assert!(ifd.compression == 32767);
+
+	let image_width: usize = ifd.image_width.try_into().unwrap();
+	let image_height: usize = ifd.image_height.try_into().unwrap();
+	let bits_per_sample: usize = ifd.bits_per_sample.into();
+	assert!(bits_per_sample == 12);
+
+	let [cfa_pattern_width, cfa_pattern_height] = ifd.cfa_pattern_dim;
+	assert!(cfa_pattern_width == 2 && cfa_pattern_height == 2);
+
+	file.seek_from_start(ifd.strip_offsets[0]).unwrap();
+	let mut image = sony_arw2_load_raw(image_width, image_height, ifd.sony_tone_curve, file).unwrap();
+
+	// Converting the bps from 12 to 14 so that ARW 2.3.1 and 2.3.5 have the same 14 bps.
+	image.iter_mut().for_each(|x| *x <<= 2);
+
+	RawImage {
+		data: image,
+		width: image_width,
+		height: image_height,
+		cfa_pattern: ifd.cfa_pattern.try_into().unwrap(),
+		maximum: (1 << 14) - 1,
+		black: SubtractBlack::CfaGrid([512, 512, 512, 512]), // TODO: Find the correct way to do this
+		transform: Transform::Horizontal,
+		camera_model: None,
+		camera_white_balance: ifd.white_balance_levels.map(|arr| arr.map(|x| x as f64)),
+		white_balance: None,
+		camera_to_rgb: None,
+	}
+}
+
+fn as_u32(buffer: &[u8], endian: Endian) -> Option<u32> {
+	Some(match endian {
+		Endian::Little => u32::from_le_bytes(buffer.try_into().ok()?),
+		Endian::Big => u32::from_be_bytes(buffer.try_into().ok()?),
+	})
+}
+
+fn as_u16(buffer: &[u8], endian: Endian) -> Option<u16> {
+	Some(match endian {
+		Endian::Little => u16::from_le_bytes(buffer.try_into().ok()?),
+		Endian::Big => u16::from_be_bytes(buffer.try_into().ok()?),
+	})
+}
+
+fn sony_arw2_load_raw<R: Read + Seek>(width: usize, height: usize, curve: CurveLookupTable, file: &mut TiffRead<R>) -> Option<Vec<u16>> {
+	let mut image = vec![0_u16; height * width];
+	let mut data = vec![0_u8; width + 1];
+
+	for row in 0..height {
+		file.read_exact(&mut data[0..width]).unwrap();
+
+		let mut column = 0;
+		let mut data_index = 0;
+
+		while column < width - 30 {
+			let data_value = as_u32(&data[data_index..][..4], file.endian()).unwrap();
+			let max = (0x7ff & data_value) as u16;
+			let min = (0x7ff & data_value >> 11) as u16;
+			let index_to_set_max = 0x0f & data_value >> 22;
+			let index_to_set_min = 0x0f & data_value >> 26;
+
+			let max_minus_min = max as i32 - min as i32;
+			let shift_by_bits = (0..4).find(|&shift| (0x80 << shift) > max_minus_min).unwrap_or(4);
+
+			let mut pixels = [0_u16; 16];
+			let mut bit = 30;
+			for (i, pixel) in pixels.iter_mut().enumerate() {
+				*pixel = match () {
+					_ if i as u32 == index_to_set_max => max,
+					_ if i as u32 == index_to_set_min => min,
+					_ => {
+						let result = as_u16(&data[(data_index + (bit >> 3))..][..2], file.endian()).unwrap();
+						let result = ((result >> (bit & 7)) & 0x07f) << shift_by_bits;
+
+						bit += 7;
+
+						(result + min).min(0x7ff)
+					}
+				};
+			}
+
+			for value in pixels {
+				image[row * width + column] = curve.get((value << 1).into()) >> 2;
+
+				// Skip between interlaced columns
+				column += 2;
+			}
+
+			// Switch to the opposite interlaced columns
+			column -= if column & 1 == 0 { 31 } else { 1 };
+
+			data_index += 16;
+		}
+	}
+
+	Some(image)
+}

libraries/rawkit/src/decoder/mod.rs

@@ -0,0 +1,3 @@
+pub mod arw1;
+pub mod arw2;
+pub mod uncompressed;

libraries/rawkit/src/decoder/uncompressed.rs

@@ -0,0 +1,67 @@
+use crate::tiff::file::TiffRead;
+use crate::tiff::tags::{BitsPerSample, BlackLevel, CfaPattern, CfaPatternDim, Compression, ImageLength, ImageWidth, RowsPerStrip, StripByteCounts, StripOffsets, Tag, WhiteBalanceRggbLevels};
+use crate::tiff::{Ifd, TiffError};
+use crate::{RawImage, SubtractBlack, Transform};
+
+use rawkit_proc_macros::Tag;
+use std::io::{Read, Seek};
+
+#[allow(dead_code)]
+#[derive(Tag)]
+struct ArwUncompressedIfd {
+	image_width: ImageWidth,
+	image_height: ImageLength,
+	rows_per_strip: RowsPerStrip,
+	bits_per_sample: BitsPerSample,
+	compression: Compression,
+	black_level: BlackLevel,
+	cfa_pattern: CfaPattern,
+	cfa_pattern_dim: CfaPatternDim,
+	strip_offsets: StripOffsets,
+	strip_byte_counts: StripByteCounts,
+	white_balance_levels: Option<WhiteBalanceRggbLevels>,
+}
+
+pub fn decode<R: Read + Seek>(ifd: Ifd, file: &mut TiffRead<R>) -> RawImage {
+	let ifd = ifd.get_value::<ArwUncompressedIfd, _>(file).unwrap();
+
+	assert!(ifd.strip_offsets.len() == ifd.strip_byte_counts.len());
+	assert!(ifd.strip_offsets.len() == 1);
+	assert!(ifd.compression == 1); // 1 is the value for uncompressed format
+
+	let image_width: usize = ifd.image_width.try_into().unwrap();
+	let image_height: usize = ifd.image_height.try_into().unwrap();
+	let rows_per_strip: usize = ifd.rows_per_strip.try_into().unwrap();
+	let bits_per_sample: usize = ifd.bits_per_sample.into();
+	let [cfa_pattern_width, cfa_pattern_height] = ifd.cfa_pattern_dim;
+	assert!(cfa_pattern_width == 2 && cfa_pattern_height == 2);
+
+	let mut image: Vec<u16> = Vec::with_capacity(image_height * image_width);
+
+	for i in 0..ifd.strip_offsets.len() {
+		file.seek_from_start(ifd.strip_offsets[i]).unwrap();
+
+		let last = i == ifd.strip_offsets.len();
+		let rows = if last { image_height % rows_per_strip } else { rows_per_strip };
+
+		for _ in 0..rows {
+			for _ in 0..image_width {
+				image.push(file.read_u16().unwrap());
+			}
+		}
+	}
+
+	RawImage {
+		data: image,
+		width: image_width,
+		height: image_height,
+		cfa_pattern: ifd.cfa_pattern.try_into().unwrap(),
+		maximum: if bits_per_sample == 16 { u16::MAX } else { (1 << bits_per_sample) - 1 },
+		black: SubtractBlack::CfaGrid(ifd.black_level),
+		transform: Transform::Horizontal,
+		camera_model: None,
+		camera_white_balance: ifd.white_balance_levels.map(|arr| arr.map(|x| x as f64)),
+		white_balance: None,
+		camera_to_rgb: None,
+	}
+}