为了更好地利用 Firebase 平台,我们最近将应用程序从 Azure 迁移到了 Google Cloud Platform。现在,我们拥有大量 firebase 函数,它们通过设置为最大 1000 Mbps 的 VPC 连接器与 Redis 实例进行通信。该功能可以正常工作,但与我们在 Azure 上使用微服务运行 Redis 查询时相比,Redis 查询在高峰时段的性能非常慢。
我们在 GCP 上运行的 Redis 实例是一个具有 25 GB RAM 的标准实例,但目前只使用了大约 13 GB。我们有大约 300,000 个键,它们都是永不过期的集合或有序集合(ttl=0)。应用程序不断向各个集合添加更多项目。
我们不使用 Redis 作为缓存,而是将其作为快速交叉和处理数据的逻辑层。所有键之间都可以进行交叉,因此,如果可能的话,聚类似乎是一场噩梦。
在高峰时段,Redis 中的简单操作PING
可能需要 60 秒以上才能响应PONG
。在非高峰时段,它仍可能需要 10 秒以上。CPU 图表看起来不错,有时会达到 100% 的峰值,但从未停留在那里。根据 ,内存碎片为 1.06,info memory
根据我所读的内容,这也应该没问题。延迟似乎也不是问题。我已经阅读了多个“如何监控 redis 性能”指南,但我仍然无法找出问题所在。
这里有人知道下一步该调查什么吗?
以下是 -command 的输出info all
。
# Server
redis_version:4.0.14
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:1a41b421362ff783
redis_mode:standalone
os:Linux 4.19.112+ x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:6.3.0
process_id:1
run_id:0f04aeebf8790dbf66e8b6b71c95157832e83804
tcp_port:6379
uptime_in_seconds:5372676
uptime_in_days:62
hz:10
lru_clock:15916574
executable:/data/redis-server
config_file:/usr/local/etc/redis/redis.conf
# Clients
connected_clients:386
client_longest_output_list:1
client_biggest_input_buf:0
blocked_clients:0
# Memory
used_memory:13755985544
used_memory_human:12.81G
used_memory_rss:14612512768
used_memory_rss_human:13.61G
used_memory_peak:13821377392
used_memory_peak_human:12.87G
used_memory_peak_perc:99.53%
used_memory_overhead:2717930060
used_memory_startup:3653752
used_memory_dataset:11038055484
used_memory_dataset_perc:80.26%
total_system_memory:35849793536
total_system_memory_human:33.39G
used_memory_lua:3240960
used_memory_lua_human:3.09M
maxmemory:26843545600
maxmemory_human:25.00G
maxmemory_policy:volatile-lru
mem_fragmentation_ratio:1.06
mem_allocator:jemalloc-4.0.3
active_defrag_running:0
lazyfree_pending_objects:0
# Persistence
loading:0
rdb_changes_since_last_save:12652002628
rdb_bgsave_in_progress:0
rdb_last_save_time:1609594635
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:123
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:717807616
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
# Stats
total_connections_received:11772107
total_commands_processed:44766456481
instantaneous_ops_per_sec:8999
total_net_input_bytes:11667388987
total_net_output_bytes:23685769242
instantaneous_input_kbps:2.23
instantaneous_output_kbps:3.19
rejected_connections:0
sync_full:1
sync_partial_ok:0
sync_partial_err:1
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
evicted_keys:0
keyspace_hits:7340540
keyspace_misses:265889
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:302420
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
# Replication
role:master
connected_slaves:1
slave0:ip=10.150.95.59,port=6379,state=online,offset=10678179325,lag=3
master_replid:f497652148f651d0c087a9d2bb33fd5f124e18a6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:10678204720
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:2684354560
repl_backlog_first_byte_offset:7993850161
repl_backlog_histlen:2684354560
# CPU
used_cpu_sys:7674.65
used_cpu_user:3521705.50
used_cpu_sys_children:18.38
used_cpu_user_children:120.26
# Commandstats
cmdstat_zadd:calls=44559021486,usec=266567931159,usec_per_call=5.98
cmdstat_get:calls=10595,usec=11200,usec_per_call=1.06
cmdstat_multi:calls=37785398,usec=21279258,usec_per_call=0.56
cmdstat_subscribe:calls=6543,usec=29469,usec_per_call=4.50
cmdstat_config:calls=241673,usec=61201855,usec_per_call=253.24
cmdstat_dbsize:calls=3618,usec=3112,usec_per_call=0.86
cmdstat_sadd:calls=16166,usec=84124,usec_per_call=5.20
cmdstat_psync:calls=1,usec=149,usec_per_call=149.00
cmdstat_bgsave:calls=1,usec=302657,usec_per_call=302657.00
cmdstat_info:calls=2399503,usec=320297314,usec_per_call=133.48
cmdstat_zrangebyscore:calls=2531456,usec=61389137799,usec_per_call=24250.53
cmdstat_zrange:calls=312,usec=20303884,usec_per_call=65076.55
cmdstat_auth:calls=1720895,usec=1254440,usec_per_call=0.73
cmdstat_slaveof:calls=80343,usec=126039,usec_per_call=1.57
cmdstat_lastsave:calls=3,usec=3,usec_per_call=1.00
cmdstat_role:calls=16196665,usec=175267620,usec_per_call=10.82
cmdstat_echo:calls=13088,usec=5588,usec_per_call=0.43
cmdstat_replconf:calls=2087395,usec=2168422,usec_per_call=1.04
cmdstat_ping:calls=13297,usec=5590,usec_per_call=0.42
cmdstat_zrem:calls=258,usec=116323,usec_per_call=450.86
cmdstat_smembers:calls=188,usec=4924728,usec_per_call=26195.36
cmdstat_select:calls=1697357,usec=1228773,usec_per_call=0.72
cmdstat_srem:calls=149,usec=15348,usec_per_call=103.01
cmdstat_exec:calls=142630025,usec=7053257948255,usec_per_call=49451.42
cmdstat_command:calls=3,usec=1355,usec_per_call=451.67
cmdstat_scan:calls=63,usec=1015832,usec_per_call=16124.32
# Cluster
cluster_enabled:0
# Keyspace
db0:keys=312817,expires=0,avg_ttl=0
# oom-prevention
oom_prevention_on:No
current_useconds_with_oom_prevention_on:0
total_useconds_with_oom_prevention_on:0
oom_prevention_threshold:30619509719
oom_prevention_threshold_human:28.52G
used_memory_total:14612512768
used_memory_total_human:13.61G
peak_used_memory_total:15259754496
peak_used_memory_total_human:14.21G
used_memory_aof:0
used_memory_aof_human:0B
used_memory_rdb:0
used_memory_rdb_human:0B