Gih's Blog

只言片语
Archive for July 2014

Strip chars from a string in Go.

2014-07-22 by gihnius, tagged as go
import (
    "fmt"
    "strings"
)

func stripchars(str, chr string) string {
    return strings.Map(func(r rune) rune {
        if strings.IndexRune(chr, r) < 0 {
            return r
        }
        return -1
    }, str)
}

To clean up old packages in quicklisp

2014-07-22 by gihnius, tagged as lisp
(map nil 'ql-dist:clean (ql-dist:all-dists))

A better way to backup redis' rdb snapshot.

2014-07-20 by gihnius, tagged as redis

I first post a relate article here: http://www.qufor.com/topics/529999c87cc1f87c30000004

Here is an update.

Basically, you can directly copy the file to a safe place, here is the redis document about persistence: http://redis.io/topics/persistence

Redis is very data backup friendly since you can copy RDB files while the database is running: the RDB is never modified once produced, and while it gets produced it uses a temporary name and is renamed into its final destination atomically using rename(2) only when the new snapshot is complete.

You can get the persistence info by running this command: 

echo 'info Persistence' | redis-cli 

You will get an output like the following

# Persistence
loading:0
rdb_changes_since_last_save:6
rdb_bgsave_in_progress:0
rdb_last_save_time:1406022719
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:0
rdb_current_bgsave_time_sec:-1
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok

See these lines: rdb_bgsave_in_progress:0 and rdb_last_bgsave_status:ok. Why I need to verify the bgsave status?

By default redis save the dataset every N seconds if there are at least M changes in the dataset, this can be changed from configuration:

save 60 1000

if redis data is changed very frequent, I'd like to get the latest snapshot, so I'd run bgsave before copying rdb file by this command:

echo bgsave | redis-cli

So here is the finally script to do the backup:

#!/bin/bash

## simple redis rdb backup script
## usage
## rdb-backup.sh rdb.path backup.dir bgsave.wait.seconds

rdb=${1:-"/var/db/redis/dump.rdb"}

backup_to=${2:-"/data/backup/redis/"}

wait=${3:-10} ## default wait for 10 seconds

test -f "$rdb" || {
    echo No rdb file found ; exit 1
}
test -d "$backup_to" || {
    echo Creating backup directory $backup_to && mkdir -p "$backup_to"
}

## launch bgsave
echo bgsave | redis-cli
echo "waiting for $wait seconds..."
sleep $wait
try=5
while [ $try -gt 0 ] ; do
    saved=$(echo 'info Persistence' | redis-cli | awk '/rdb_bgsave_in_progress:0/{print "saved"}')
    ok=$(echo 'info Persistence' | redis-cli | awk '/rdb_last_bgsave_status:ok/{print "ok"}')
    if [[ "$saved" = "saved" ]] && [[ "$ok" = "ok" ]] ; then
        cp "$rdb" "$backup_to"
        if [ $? = 0 ] ; then
            echo "redis rdb $rdb copied to $backup_to ."
            exit 0
        else 
            echo ">> Failed to copy $rdb to $backup_to !"
        fi
    fi
    try=$((try - 1))
    echo "redis maybe busy, waiting and retry in 5s..."
    sleep 5
done
exit 1

Available on github: https://github.com/gihnius/rdb-backup

Counting online users with Redis and Go

2014-07-20 by gihnius, tagged as go

an alternative way if you do not need to use websocket to do realtime counting.

only counting last 5 minutes, here assuming a user will stay online for five minutes after login/landing

var last_n_minutes = 5

// easily to change to yours
func redis_key_prefix() string {
    return App.Name + ":" + App.Env + ":"
}

func redis_key(suffix string) string {
    return redis_key_prefix() + suffix
}

// online key for online users every minute
// online_users_minute_5
// online_users_minute_6 // 5th and 6th minutes online users
func online_key(minute_suffix string) string {
    key := "online_users_minute_" + minute_suffix
    return redis_key(key)
}

// the key for THIS minute
func current_key() string {
    key := strconv.Itoa(time.Now().Minute())
    return online_key(key)
}

// return keys of last n minutes online users
func keys_in_last_n_minutes(n int) []string {
    now := time.Now()
    var res []string
    for i := 0; i < n ; i++ {
        ago := now.Add(-time.Duration(i) * time.Minute).Minute()
        res = append(res, online_key(strconv.Itoa(ago)))
    }
    return res
}

// add a online user to the set.
// call this operation from a ajax long pull is recommended, so
// do not need to write to redis every time user click/open a page.
func add_online_username(name string) {
    new_key := false
    key := current_key()
    if ok, _ := Redis.Exists(key); ok == false {
        new_key = true
    }
    Redis.Sadd(key, []byte(name))
    if new_key {
        // assuming a user will be offline after last_n_minutes
        expiredin := int64((last_n_minutes+1)*60)
        Redis.Expire(key, expiredin)
    }
}

// the online usernames
func online_usernames() []string {
    keys := keys_in_last_n_minutes(last_n_minutes)
    users, err := Redis.Sunion(keys...)
    if err != nil {
        return nil
    }
    var res []string
    for _, u := range users {
        res = append(res, string(u))
    }
    return res
}

// counting how many online users
// just do it from redis
func online_users_count() int {
    current_online_key := redis_key("online_users_current")
    keys := keys_in_last_n_minutes(last_n_minutes)
    Redis.Sunionstore(current_online_key, keys...)
    n, err := Redis.Scard(current_online_key)
    if err != nil {
        return -1
    }
    // go set_online_max(n)
    set_online_max(n)
    return n
}

// the max value of online users
func set_online_max(curr int) {
    max_online_key := redis_key("online_users_max")
    orig, _ := Redis.Get(max_online_key)
    n, _ := strconv.Atoi(string(orig))
    if curr > n {
        Redis.Set(max_online_key, []byte(strconv.Itoa(curr)))
    }
}

func get_online_max() int {
    max_online_key := redis_key("online_users_max")
    orig, _ := Redis.Get(max_online_key)
    n, err := strconv.Atoi(string(orig))
    if err != nil {
        return -1
    }
    return n
}

The redis package using here is: github.com/hoisie/redis, to install it to GOPATH:

go get -u github.com/hoisie/redis

Get the code and a full example, please check out: https://github.com/gihnius/redis_online_counter

A nice ORM library in common lisp

2014-07-20 by gihnius, tagged as lisp

Crane project home page: http://eudoxia0.github.io/crane/

(filter 'user) ;; Returns everything

(filter 'user :name "Eudoxia")(filter 'user (:> :age 21))

;; Returns a single object
(single 'user :name "Eudoxia");;Throws an error ifthis returns more
;; than one object(single!'user (:< age 35))

;; t if a match exists, nil otherwise
(exists 'user :name "Eudoxia");;Ifthis record doesn't exist create it
(get-or-create 'user :name "Eudoxia":age 19)