kernel-aes67/drivers/md/dm-vdo/slab-depot.h

602 lines
21 KiB
C

/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright 2023 Red Hat
*/
#ifndef VDO_SLAB_DEPOT_H
#define VDO_SLAB_DEPOT_H
#include <linux/atomic.h>
#include <linux/dm-kcopyd.h>
#include <linux/list.h>
#include "numeric.h"
#include "admin-state.h"
#include "completion.h"
#include "data-vio.h"
#include "encodings.h"
#include "physical-zone.h"
#include "priority-table.h"
#include "recovery-journal.h"
#include "statistics.h"
#include "types.h"
#include "vio.h"
#include "wait-queue.h"
/*
* A slab_depot is responsible for managing all of the slabs and block allocators of a VDO. It has
* a single array of slabs in order to eliminate the need for additional math in order to compute
* which physical zone a PBN is in. It also has a block_allocator per zone.
*
* Each physical zone has a single dedicated queue and thread for performing all updates to the
* slabs assigned to that zone. The concurrency guarantees of this single-threaded model allow the
* code to omit more fine-grained locking for the various slab structures. Each physical zone
* maintains a separate copy of the slab summary to remove the need for explicit locking on that
* structure as well.
*
* Load operations must be performed on the admin thread. Normal operations, such as allocations
* and reference count updates, must be performed on the appropriate physical zone thread. Requests
* from the recovery journal to commit slab journal tail blocks must be scheduled from the recovery
* journal thread to run on the appropriate physical zone thread. Save operations must be launched
* from the same admin thread as the original load operation.
*/
enum {
/* The number of vios in the vio pool is proportional to the throughput of the VDO. */
BLOCK_ALLOCATOR_VIO_POOL_SIZE = 128,
};
/*
* Represents the possible status of a block.
*/
enum reference_status {
RS_FREE, /* this block is free */
RS_SINGLE, /* this block is singly-referenced */
RS_SHARED, /* this block is shared */
RS_PROVISIONAL /* this block is provisionally allocated */
};
struct vdo_slab;
struct journal_lock {
u16 count;
sequence_number_t recovery_start;
};
struct slab_journal {
/* A waiter object for getting a VIO pool entry */
struct vdo_waiter resource_waiter;
/* A waiter object for updating the slab summary */
struct vdo_waiter slab_summary_waiter;
/* A waiter object for getting a vio with which to flush */
struct vdo_waiter flush_waiter;
/* The queue of VIOs waiting to make an entry */
struct vdo_wait_queue entry_waiters;
/* The parent slab reference of this journal */
struct vdo_slab *slab;
/* Whether a tail block commit is pending */
bool waiting_to_commit;
/* Whether the journal is updating the slab summary */
bool updating_slab_summary;
/* Whether the journal is adding entries from the entry_waiters queue */
bool adding_entries;
/* Whether a partial write is in progress */
bool partial_write_in_progress;
/* The oldest block in the journal on disk */
sequence_number_t head;
/* The oldest block in the journal which may not be reaped */
sequence_number_t unreapable;
/* The end of the half-open interval of the active journal */
sequence_number_t tail;
/* The next journal block to be committed */
sequence_number_t next_commit;
/* The tail sequence number that is written in the slab summary */
sequence_number_t summarized;
/* The tail sequence number that was last summarized in slab summary */
sequence_number_t last_summarized;
/* The sequence number of the recovery journal lock */
sequence_number_t recovery_lock;
/*
* The number of entries which fit in a single block. Can't use the constant because unit
* tests change this number.
*/
journal_entry_count_t entries_per_block;
/*
* The number of full entries which fit in a single block. Can't use the constant because
* unit tests change this number.
*/
journal_entry_count_t full_entries_per_block;
/* The recovery journal of the VDO (slab journal holds locks on it) */
struct recovery_journal *recovery_journal;
/* The statistics shared by all slab journals in our physical zone */
struct slab_journal_statistics *events;
/* A list of the VIO pool entries for outstanding journal block writes */
struct list_head uncommitted_blocks;
/*
* The current tail block header state. This will be packed into the block just before it
* is written.
*/
struct slab_journal_block_header tail_header;
/* A pointer to a block-sized buffer holding the packed block data */
struct packed_slab_journal_block *block;
/* The number of blocks in the on-disk journal */
block_count_t size;
/* The number of blocks at which to start pushing reference blocks */
block_count_t flushing_threshold;
/* The number of blocks at which all reference blocks should be writing */
block_count_t flushing_deadline;
/* The number of blocks at which to wait for reference blocks to write */
block_count_t blocking_threshold;
/* The number of blocks at which to scrub the slab before coming online */
block_count_t scrubbing_threshold;
/* This list entry is for block_allocator to keep a queue of dirty journals */
struct list_head dirty_entry;
/* The lock for the oldest unreaped block of the journal */
struct journal_lock *reap_lock;
/* The locks for each on disk block */
struct journal_lock *locks;
};
/*
* Reference_block structure
*
* Blocks are used as a proxy, permitting saves of partial refcounts.
*/
struct reference_block {
/* This block waits on the ref_counts to tell it to write */
struct vdo_waiter waiter;
/* The slab to which this reference_block belongs */
struct vdo_slab *slab;
/* The number of references in this block that represent allocations */
block_size_t allocated_count;
/* The slab journal block on which this block must hold a lock */
sequence_number_t slab_journal_lock;
/* The slab journal block which should be released when this block is committed */
sequence_number_t slab_journal_lock_to_release;
/* The point up to which each sector is accurate on disk */
struct journal_point commit_points[VDO_SECTORS_PER_BLOCK];
/* Whether this block has been modified since it was written to disk */
bool is_dirty;
/* Whether this block is currently writing */
bool is_writing;
};
/* The search_cursor represents the saved position of a free block search. */
struct search_cursor {
/* The reference block containing the current search index */
struct reference_block *block;
/* The position at which to start searching for the next free counter */
slab_block_number index;
/* The position just past the last valid counter in the current block */
slab_block_number end_index;
/* A pointer to the first reference block in the slab */
struct reference_block *first_block;
/* A pointer to the last reference block in the slab */
struct reference_block *last_block;
};
enum slab_rebuild_status {
VDO_SLAB_REBUILT,
VDO_SLAB_REPLAYING,
VDO_SLAB_REQUIRES_SCRUBBING,
VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING,
VDO_SLAB_REBUILDING,
};
/*
* This is the type declaration for the vdo_slab type. A vdo_slab currently consists of a run of
* 2^23 data blocks, but that will soon change to dedicate a small number of those blocks for
* metadata storage for the reference counts and slab journal for the slab.
*
* A reference count is maintained for each physical block number. The vast majority of blocks have
* a very small reference count (usually 0 or 1). For references less than or equal to MAXIMUM_REFS
* (254) the reference count is stored in counters[pbn].
*/
struct vdo_slab {
/* A list entry to queue this slab in a block_allocator list */
struct list_head allocq_entry;
/* The struct block_allocator that owns this slab */
struct block_allocator *allocator;
/* The journal for this slab */
struct slab_journal journal;
/* The slab number of this slab */
slab_count_t slab_number;
/* The offset in the allocator partition of the first block in this slab */
physical_block_number_t start;
/* The offset of the first block past the end of this slab */
physical_block_number_t end;
/* The starting translated PBN of the slab journal */
physical_block_number_t journal_origin;
/* The starting translated PBN of the reference counts */
physical_block_number_t ref_counts_origin;
/* The administrative state of the slab */
struct admin_state state;
/* The status of the slab */
enum slab_rebuild_status status;
/* Whether the slab was ever queued for scrubbing */
bool was_queued_for_scrubbing;
/* The priority at which this slab has been queued for allocation */
u8 priority;
/* Fields beyond this point are the reference counts for the data blocks in this slab. */
/* The size of the counters array */
u32 block_count;
/* The number of free blocks */
u32 free_blocks;
/* The array of reference counts */
vdo_refcount_t *counters; /* use vdo_allocate() to align data ptr */
/* The saved block pointer and array indexes for the free block search */
struct search_cursor search_cursor;
/* A list of the dirty blocks waiting to be written out */
struct vdo_wait_queue dirty_blocks;
/* The number of blocks which are currently writing */
size_t active_count;
/* A waiter object for updating the slab summary */
struct vdo_waiter summary_waiter;
/* The latest slab journal for which there has been a reference count update */
struct journal_point slab_journal_point;
/* The number of reference count blocks */
u32 reference_block_count;
/* reference count block array */
struct reference_block *reference_blocks;
};
enum block_allocator_drain_step {
VDO_DRAIN_ALLOCATOR_START,
VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER,
VDO_DRAIN_ALLOCATOR_STEP_SLABS,
VDO_DRAIN_ALLOCATOR_STEP_SUMMARY,
VDO_DRAIN_ALLOCATOR_STEP_FINISHED,
};
struct slab_scrubber {
/* The queue of slabs to scrub first */
struct list_head high_priority_slabs;
/* The queue of slabs to scrub once there are no high_priority_slabs */
struct list_head slabs;
/* The queue of VIOs waiting for a slab to be scrubbed */
struct vdo_wait_queue waiters;
/*
* The number of slabs that are unrecovered or being scrubbed. This field is modified by
* the physical zone thread, but is queried by other threads.
*/
slab_count_t slab_count;
/* The administrative state of the scrubber */
struct admin_state admin_state;
/* Whether to only scrub high-priority slabs */
bool high_priority_only;
/* The slab currently being scrubbed */
struct vdo_slab *slab;
/* The vio for loading slab journal blocks */
struct vio vio;
};
/* A sub-structure for applying actions in parallel to all an allocator's slabs. */
struct slab_actor {
/* The number of slabs performing a slab action */
slab_count_t slab_action_count;
/* The method to call when a slab action has been completed by all slabs */
vdo_action_fn callback;
};
/* A slab_iterator is a structure for iterating over a set of slabs. */
struct slab_iterator {
struct vdo_slab **slabs;
struct vdo_slab *next;
slab_count_t end;
slab_count_t stride;
};
/*
* The slab_summary provides hints during load and recovery about the state of the slabs in order
* to avoid the need to read the slab journals in their entirety before a VDO can come online.
*
* The information in the summary for each slab includes the rough number of free blocks (which is
* used to prioritize scrubbing), the cleanliness of a slab (so that clean slabs containing free
* space will be used on restart), and the location of the tail block of the slab's journal.
*
* The slab_summary has its own partition at the end of the volume which is sized to allow for a
* complete copy of the summary for each of up to 16 physical zones.
*
* During resize, the slab_summary moves its backing partition and is saved once moved; the
* slab_summary is not permitted to overwrite the previous recovery journal space.
*
* The slab_summary does not have its own version information, but relies on the VDO volume version
* number.
*/
/*
* A slab status is a very small structure for use in determining the ordering of slabs in the
* scrubbing process.
*/
struct slab_status {
slab_count_t slab_number;
bool is_clean;
u8 emptiness;
};
struct slab_summary_block {
/* The block_allocator to which this block belongs */
struct block_allocator *allocator;
/* The index of this block in its zone's summary */
block_count_t index;
/* Whether this block has a write outstanding */
bool writing;
/* Ring of updates waiting on the outstanding write */
struct vdo_wait_queue current_update_waiters;
/* Ring of updates waiting on the next write */
struct vdo_wait_queue next_update_waiters;
/* The active slab_summary_entry array for this block */
struct slab_summary_entry *entries;
/* The vio used to write this block */
struct vio vio;
/* The packed entries, one block long, backing the vio */
char *outgoing_entries;
};
/*
* The statistics for all the slab summary zones owned by this slab summary. These fields are all
* mutated only by their physical zone threads, but are read by other threads when gathering
* statistics for the entire depot.
*/
struct atomic_slab_summary_statistics {
/* Number of blocks written */
atomic64_t blocks_written;
};
struct block_allocator {
struct vdo_completion completion;
/* The slab depot for this allocator */
struct slab_depot *depot;
/* The nonce of the VDO */
nonce_t nonce;
/* The physical zone number of this allocator */
zone_count_t zone_number;
/* The thread ID for this allocator's physical zone */
thread_id_t thread_id;
/* The number of slabs in this allocator */
slab_count_t slab_count;
/* The number of the last slab owned by this allocator */
slab_count_t last_slab;
/* The reduced priority level used to preserve unopened slabs */
unsigned int unopened_slab_priority;
/* The state of this allocator */
struct admin_state state;
/* The actor for applying an action to all slabs */
struct slab_actor slab_actor;
/* The slab from which blocks are currently being allocated */
struct vdo_slab *open_slab;
/* A priority queue containing all slabs available for allocation */
struct priority_table *prioritized_slabs;
/* The slab scrubber */
struct slab_scrubber scrubber;
/* What phase of the close operation the allocator is to perform */
enum block_allocator_drain_step drain_step;
/*
* These statistics are all mutated only by the physical zone thread, but are read by other
* threads when gathering statistics for the entire depot.
*/
/*
* The count of allocated blocks in this zone. Not in block_allocator_statistics for
* historical reasons.
*/
u64 allocated_blocks;
/* Statistics for this block allocator */
struct block_allocator_statistics statistics;
/* Cumulative statistics for the slab journals in this zone */
struct slab_journal_statistics slab_journal_statistics;
/* Cumulative statistics for the reference counters in this zone */
struct ref_counts_statistics ref_counts_statistics;
/*
* This is the head of a queue of slab journals which have entries in their tail blocks
* which have not yet started to commit. When the recovery journal is under space pressure,
* slab journals which have uncommitted entries holding a lock on the recovery journal head
* are forced to commit their blocks early. This list is kept in order, with the tail
* containing the slab journal holding the most recent recovery journal lock.
*/
struct list_head dirty_slab_journals;
/* The vio pool for reading and writing block allocator metadata */
struct vio_pool *vio_pool;
/* The dm_kcopyd client for erasing slab journals */
struct dm_kcopyd_client *eraser;
/* Iterator over the slabs to be erased */
struct slab_iterator slabs_to_erase;
/* The portion of the slab summary managed by this allocator */
/* The state of the slab summary */
struct admin_state summary_state;
/* The number of outstanding summary writes */
block_count_t summary_write_count;
/* The array (owned by the blocks) of all entries */
struct slab_summary_entry *summary_entries;
/* The array of slab_summary_blocks */
struct slab_summary_block *summary_blocks;
};
enum slab_depot_load_type {
VDO_SLAB_DEPOT_NORMAL_LOAD,
VDO_SLAB_DEPOT_RECOVERY_LOAD,
VDO_SLAB_DEPOT_REBUILD_LOAD
};
struct slab_depot {
zone_count_t zone_count;
zone_count_t old_zone_count;
struct vdo *vdo;
struct slab_config slab_config;
struct action_manager *action_manager;
physical_block_number_t first_block;
physical_block_number_t last_block;
physical_block_number_t origin;
/* slab_size == (1 << slab_size_shift) */
unsigned int slab_size_shift;
/* Determines how slabs should be queued during load */
enum slab_depot_load_type load_type;
/* The state for notifying slab journals to release recovery journal */
sequence_number_t active_release_request;
sequence_number_t new_release_request;
/* State variables for scrubbing complete handling */
atomic_t zones_to_scrub;
/* Array of pointers to individually allocated slabs */
struct vdo_slab **slabs;
/* The number of slabs currently allocated and stored in 'slabs' */
slab_count_t slab_count;
/* Array of pointers to a larger set of slabs (used during resize) */
struct vdo_slab **new_slabs;
/* The number of slabs currently allocated and stored in 'new_slabs' */
slab_count_t new_slab_count;
/* The size that 'new_slabs' was allocated for */
block_count_t new_size;
/* The last block before resize, for rollback */
physical_block_number_t old_last_block;
/* The last block after resize, for resize */
physical_block_number_t new_last_block;
/* The statistics for the slab summary */
struct atomic_slab_summary_statistics summary_statistics;
/* The start of the slab summary partition */
physical_block_number_t summary_origin;
/* The number of bits to shift to get a 7-bit fullness hint */
unsigned int hint_shift;
/* The slab summary entries for all of the zones the partition can hold */
struct slab_summary_entry *summary_entries;
/* The block allocators for this depot */
struct block_allocator allocators[];
};
struct reference_updater;
bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab,
physical_block_number_t pbn,
enum journal_operation operation,
bool increment,
struct journal_point *recovery_point,
struct vdo_completion *parent);
int __must_check vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot,
physical_block_number_t pbn,
enum journal_operation operation);
static inline struct block_allocator *vdo_as_block_allocator(struct vdo_completion *completion)
{
vdo_assert_completion_type(completion, VDO_BLOCK_ALLOCATOR_COMPLETION);
return container_of(completion, struct block_allocator, completion);
}
int __must_check vdo_acquire_provisional_reference(struct vdo_slab *slab,
physical_block_number_t pbn,
struct pbn_lock *lock);
int __must_check vdo_allocate_block(struct block_allocator *allocator,
physical_block_number_t *block_number_ptr);
int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator,
struct vdo_waiter *waiter);
void vdo_modify_reference_count(struct vdo_completion *completion,
struct reference_updater *updater);
int __must_check vdo_release_block_reference(struct block_allocator *allocator,
physical_block_number_t pbn);
void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion);
void vdo_dump_block_allocator(const struct block_allocator *allocator);
int __must_check vdo_decode_slab_depot(struct slab_depot_state_2_0 state,
struct vdo *vdo,
struct partition *summary_partition,
struct slab_depot **depot_ptr);
void vdo_free_slab_depot(struct slab_depot *depot);
struct slab_depot_state_2_0 __must_check vdo_record_slab_depot(const struct slab_depot *depot);
int __must_check vdo_allocate_reference_counters(struct slab_depot *depot);
struct vdo_slab * __must_check vdo_get_slab(const struct slab_depot *depot,
physical_block_number_t pbn);
u8 __must_check vdo_get_increment_limit(struct slab_depot *depot,
physical_block_number_t pbn);
bool __must_check vdo_is_physical_data_block(const struct slab_depot *depot,
physical_block_number_t pbn);
block_count_t __must_check vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot);
block_count_t __must_check vdo_get_slab_depot_data_blocks(const struct slab_depot *depot);
void vdo_get_slab_depot_statistics(const struct slab_depot *depot,
struct vdo_statistics *stats);
void vdo_load_slab_depot(struct slab_depot *depot,
const struct admin_state_code *operation,
struct vdo_completion *parent, void *context);
void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot,
enum slab_depot_load_type load_type,
struct vdo_completion *parent);
void vdo_update_slab_depot_size(struct slab_depot *depot);
int __must_check vdo_prepare_to_grow_slab_depot(struct slab_depot *depot,
const struct partition *partition);
void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent);
void vdo_abandon_new_slabs(struct slab_depot *depot);
void vdo_drain_slab_depot(struct slab_depot *depot,
const struct admin_state_code *operation,
struct vdo_completion *parent);
void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent);
void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot,
sequence_number_t recovery_block_number);
void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot,
struct vdo_completion *parent);
void vdo_dump_slab_depot(const struct slab_depot *depot);
#endif /* VDO_SLAB_DEPOT_H */