搞懂Redis RDB和AOF持久化及工作原理

127.0.0.1:> save

OK
(1.14s)

:M  Apr ::51.948 * DB saved on disk

127.0.0.1:> bgsave

Background saving started

:M  Apr ::40.312 * Background saving started by pid

:C  Apr ::40.314 * DB saved on disk

:M  Apr ::40.317 * Background saving terminated with success

################################ SNAPSHOTTING  ################################

#

# Save the DB on disk:

#

#   save <seconds> <changes>

#

#   Will save the DB if both the given number of seconds and the given

#   number of write operations against the DB occurred.

#

#   In the example below the behaviour will be to save:

#   after  sec ( min) if at least  key changed

#   after  sec ( min) if at least  keys changed

#   after  sec if at least  keys changed

#

#   Note: you can disable saving completely by commenting out all "save" lines.

#

#   It is also possible to remove all the previously configured save

#   points by adding a save directive with a single empty string argument

#   like in the following example:

#

#   save ""

save 900 1

save 300 10

save 60 10000

# By default Redis will stop accepting writes if RDB snapshots are enabled

# (at least one save point) and the latest background save failed.

# This will make the user aware (in a hard way) that data is not persisting

# on disk properly, otherwise chances are that no one will notice and some

# disaster will happen.

#

# If the background saving process will start working again Redis will

# automatically allow writes again.

#

# However if you have setup your proper monitoring of the Redis server

# and persistence, you may want to disable this feature so that Redis will

# continue to work as usual even if there are problems with disk,

# permissions, and so forth.

stop-writes-on-bgsave-error yes

# Compress string objects using LZF when dump .rdb databases?

# For default that's set to 'yes' as it's almost always a win.

# If you want to save some CPU in the saving child set it to 'no' but

# the dataset will likely be bigger if you have compressible values or keys.

rdbcompression yes

# Since version  of RDB a CRC64 checksum is placed at the end of the file.

# This makes the format more resistant to corruption but there is a performance

# hit to pay (around %) when saving and loading RDB files, so you can disable it

# for maximum performances.

#

# RDB files created with checksum disabled have a checksum of zero that will

# tell the loading code to skip the check.

rdbchecksum yes

# The filename where to dump the DB

dbfilename dump.rdb

# The working directory.

#

# The DB will be written inside this directory, with the filename specified

# above using the 'dbfilename' configuration directive.

#

# The Append Only File will also be created inside this directory.

#

# Note that you must specify a directory here, not a file name.

dir /usr/local/var/db/redis/

struct redisServer {

    ....

    struct saveparam *saveparams;   /* Save points array for RDB */

    ...

};

struct saveparam {

    time_t seconds;

    int changes;

};

/* This is our timer interrupt, called server.hz times per second.

 * Here is where we do a number of things that need to be done asynchronously.

 * For instance:

 *

 * - Active expired keys collection (it is also performed in a lazy way on

 *   lookup).

 * - Software watchdog.

 * - Update some statistic.

 * - Incremental rehashing of the DBs hash tables.

 * - Triggering BGSAVE / AOF rewrite, and handling of terminated children.

 * - Clients timeout of different kinds.

 * - Replication reconnection.

 * - Many more...

 *

 * Everything directly called here will be called server.hz times per second,

 * so in order to throttle execution of things we want to do less frequently

 * a macro is used: run_with_period(milliseconds) { .... }

 */

int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {

/* If there is not a background saving/rewrite in progress check if

         * we have to save/rewrite now */

         for (j = ; j < server.saveparamslen; j++) {

            struct saveparam *sp = server.saveparams+j;

            /* Save if we reached the given amount of changes,

             * the given amount of seconds, and if the latest bgsave was

             * successful or if, in case of an error, at least

             * CONFIG_BGSAVE_RETRY_DELAY seconds already elapsed. */

            if (server.dirty >= sp->changes &&

                server.unixtime-server.lastsave > sp->seconds &&

                (server.unixtime-server.lastbgsave_try >

                 CONFIG_BGSAVE_RETRY_DELAY ||

                 server.lastbgsave_status == C_OK))

            {

                serverLog(LL_NOTICE,"%d changes in %d seconds. Saving...",

                    sp->changes, (int)sp->seconds);

                rdbSaveBackground(server.rdb_filename);

                break;

            }

         }

int rdbSaveBackground(char *filename) {

    pid_t childpid;

    long long start;

    if (server.aof_child_pid != - || server.rdb_child_pid != -) return C_ERR;

    server.dirty_before_bgsave = server.dirty;

    server.lastbgsave_try = time(NULL);

    start = ustime();

    if ((childpid = fork()) == ) {

        int retval;

        /* Child */

        closeListeningSockets();

        redisSetProcTitle("redis-rdb-bgsave");

        retval = rdbSave(filename);

        if (retval == C_OK) {

            size_t private_dirty = zmalloc_get_private_dirty();

            if (private_dirty) {

                serverLog(LL_NOTICE,

                    "RDB: %zu MB of memory used by copy-on-write",

                    private_dirty/(*));

            }

        }

        exitFromChild((retval == C_OK) ?  : );

    }

rdb -c memory dump.rdb > testMjx.csv

database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry

,string,mjx3,,string,,,

,string,mjx5,,string,,,

,string,mjx2,,string,,,

,string,mjx,,string,,,

,string,mjx4,,string,,,

127.0.0.1:> bgrewriteoaf

OK

############################## APPEND ONLY MODE ###############################

# By default Redis asynchronously dumps the dataset on disk. This mode is

# good enough in many applications, but an issue with the Redis process or

# a power outage may result into a few minutes of writes lost (depending on

# the configured save points).

#

# The Append Only File is an alternative persistence mode that provides

# much better durability. For instance using the default data fsync policy

# (see later in the config file) Redis can lose just one second of writes in a

# dramatic event like a server power outage, or a single write if something

# wrong with the Redis process itself happens, but the operating system is

# still running correctly.

#

# AOF and RDB persistence can be enabled at the same time without problems.

# If the AOF is enabled on startup Redis will load the AOF, that is the file

# with the better durability guarantees.

#

# Please check http://redis.io/topics/persistence for more information.

appendonly no

# The name of the append only file (default: "appendonly.aof")

appendfilename "appendonly.aof"

# The fsync() call tells the Operating System to actually write data on disk

# instead of waiting for more data in the output buffer. Some OS will really flush

# data on disk, some other OS will just try to do it ASAP.

#

# Redis supports three different modes:

#

# no: don't fsync, just let the OS flush the data when it wants. Faster.

# always: fsync after every write to the append only log. Slow, Safest.

# everysec: fsync only one time every second. Compromise.

#

# The default is "everysec", as that's usually the right compromise between

# speed and data safety. It's up to you to understand if you can relax this to

# "no" that will let the operating system flush the output buffer when

# it wants, for better performances (but if you can live with the idea of

# some data loss consider the default persistence mode that's snapshotting),

# or on the contrary, use "always" that's very slow but a bit safer than

# everysec.

#

# More details please check the following article:

# http://antirez.com/post/redis-persistence-demystified.html

#

# If unsure, use "everysec".

# appendfsync always

appendfsync everysec

# appendfsync no

# When the AOF fsync policy is set to always or everysec, and a background

# saving process (a background save or AOF log background rewriting) is

# performing a lot of I/O against the disk, in some Linux configurations

# Redis may block too long on the fsync() call. Note that there is no fix for

# this currently, as even performing fsync in a different thread will block

# our synchronous write() call.

#

# In order to mitigate this problem it's possible to use the following option

# that will prevent fsync() from being called in the main process while a

# BGSAVE or BGREWRITEAOF is in progress.

#

# This means that while another child is saving, the durability of Redis is

# the same as "appendfsync none". In practical terms, this means that it is

# possible to lose up to  seconds of log in the worst scenario (with the

# default Linux settings).

#

# If you have latency problems turn this to "yes". Otherwise leave it as

# "no" that is the safest pick from the point of view of durability.

no-appendfsync-on-rewrite no

# Automatic rewrite of the append only file.

# Redis is able to automatically rewrite the log file implicitly calling

# BGREWRITEAOF when the AOF log size grows by the specified percentage.

#

# This is how it works: Redis remembers the size of the AOF file after the

# latest rewrite (if no rewrite has happened since the restart, the size of

# the AOF at startup is used).

#

# This base size is compared to the current size. If the current size is

# bigger than the specified percentage, the rewrite is triggered. Also

# you need to specify a minimal size for the AOF file to be rewritten, this

# is useful to avoid rewriting the AOF file even if the percentage increase

# is reached but it is still pretty small.

#

# Specify a percentage of zero in order to disable the automatic AOF

# rewrite feature.

auto-aof-rewrite-percentage 100

auto-aof-rewrite-min-size 64mb

# An AOF file may be found to be truncated at the end during the Redis

# startup process, when the AOF data gets loaded back into memory.

# This may happen when the system where Redis is running

# crashes, especially when an ext4 filesystem is mounted without the

# data=ordered option (however this can't happen when Redis itself

# crashes or aborts but the operating system still works correctly).

#

# Redis can either exit with an error when this happens, or load as much

# data as possible (the default now) and start if the AOF file is found

# to be truncated at the end. The following option controls this behavior.

#

# If aof-load-truncated is set to yes, a truncated AOF file is loaded and

# the Redis server starts emitting a log to inform the user of the event.

# Otherwise if the option is set to no, the server aborts with an error

# and refuses to start. When the option is set to no, the user requires

# to fix the AOF file using the "redis-check-aof" utility before to restart

# the server.

#

# Note that if the AOF file will be found to be corrupted in the middle

# the server will still exit with an error. This option only applies when

# Redis will try to read more data from the AOF file but not enough bytes

# will be found.

aof-load-truncated yes

# When rewriting the AOF file, Redis is able to use an RDB preamble in the

# AOF file for faster rewrites and recoveries. When this option is turned

# on the rewritten AOF file is composed of two different stanzas:

#

#   [RDB file][AOF tail]

#

# When loading Redis recognizes that the AOF file starts with the "REDIS"

# string and loads the prefixed RDB file, and continues loading the AOF

# tail.

#

# This is currently turned off by default in order to avoid the surprise

# of a format change, but will at some point be used as the default.

aof-use-rdb-preamble no

/* This is how rewriting of the append only file in background works:

 *

 * 1) The user calls BGREWRITEAOF

 * 2) Redis calls this function, that forks():

 *    2a) the child rewrite the append only file in a temp file.

 *    2b) the parent accumulates differences in server.aof_rewrite_buf.

 * 3) When the child finished '2a' exists.

 * 4) The parent will trap the exit code, if it's OK, will append the

 *    data accumulated into server.aof_rewrite_buf into the temp file, and

 *    finally will rename(2) the temp file in the actual file name.

 *    The the new file is reopened as the new append only file. Profit!

 */

int rewriteAppendOnlyFileBackground(void) {

    pid_t childpid;

    long long start;

    if (server.aof_child_pid != - || server.rdb_child_pid != -) return C_ERR;

    if (aofCreatePipes() != C_OK) return C_ERR;

    start = ustime();

    if ((childpid = fork()) == ) {

        char tmpfile[];

        /* Child */

        closeListeningSockets();

        redisSetProcTitle("redis-aof-rewrite");

        snprintf(tmpfile,,"temp-rewriteaof-bg-%d.aof", (int) getpid());

        if (rewriteAppendOnlyFile(tmpfile) == C_OK) {

            size_t private_dirty = zmalloc_get_private_dirty();

            if (private_dirty) {

                serverLog(LL_NOTICE,

                    "AOF rewrite: %zu MB of memory used by copy-on-write",

                    private_dirty/(*));

            }

            exitFromChild();

        } else {

            exitFromChild();

        }

    }
...
...

127.0.0.1:> CONFIG SET appendonly yes

OK

:M  Apr ::53.940 * Background append only file rewriting started by pid

:M  Apr ::53.964 * AOF rewrite child asks to stop sending diffs.

:C  Apr ::53.965 * Parent agreed to stop sending diffs. Finalizing AOF...

:C  Apr ::53.965 * Concatenating 0.00 MB of AOF diff received from parent.

:C  Apr ::53.966 * SYNC append only file rewrite performed

:M  Apr ::53.996 * Background AOF rewrite terminated with success

:M  Apr ::53.996 * Residual parent diff successfully flushed to the rewritten AOF (0.00 MB)

:M  Apr ::53.997 * Background AOF rewrite finished successfully

127.0.0.1:> CONFIG SET appendonly yes

OK

127.0.0.1:> set miao

OK

127.0.0.1:> set miao

OK

127.0.0.1:> lpush mlist

(integer)

127.0.0.1:> lpush mlist

(integer)

127.0.0.1:> lpush mlist

(integer)

127.0.0.1:> keys *

) "miao"

) "mlist"

*

$

SELECT

$

*

$

SET

$

miao

$

*

$

SELECT

$

*

$

lpush

$

mlist

$

*

$

lpush

$

mlist

$

*

$

lpush

$

mlist

$

127.0.0.1:> bgrewriteaof

Background append only file rewriting started

:M  Apr ::31.017 *  changes in  seconds. Saving...

:M  Apr ::31.017 * Background saving started by pid

:C  Apr ::31.020 * DB saved on disk

:M  Apr ::31.120 * Background saving terminated with success

:M  Apr ::49.409 * Background append only file rewriting started by pid

:M  Apr ::49.433 * AOF rewrite child asks to stop sending diffs.

:C  Apr ::49.433 * Parent agreed to stop sending diffs. Finalizing AOF...

:C  Apr ::49.434 * Concatenating 0.00 MB of AOF diff received from parent.

:C  Apr ::49.434 * SYNC append only file rewrite performed

:M  Apr ::49.533 * Background AOF rewrite terminated with success

:M  Apr ::49.533 * Residual parent diff successfully flushed to the rewritten AOF (0.00 MB)

:M  Apr ::49.534 * Background AOF rewrite finished successfully

*

$

SELECT

$

*

$

SET

$

miao

$

*

$

RPUSH

$

mlist

$

$

$