Nova Configuration
Disk_cachemodes = "Network=writeback" (enabled)
Change to Disk_cachemodes = "Network=none" (off)
- Ceph Configuration
- Open Ceph RBD Cache
Client
Rbd_cache = True
Rbd_cache_writethrough_until_flush = True
Admin_socket =/var/run/ceph/guests/$cluster-$type. $id. $pid. $cctid. Asok
Log_file =/var/log/qemu/qemu-guest-$pid. log
Rbd_concurrent_management_ops = 20
- Close Ceph RBD Cache
[Client]
#rbd_cache = True
#rbd_cache_writethrough_until_flush = True
Admin_socket =/var/run/ceph/guests/$cluster-$type. $id. $pid. $cctid. Asok
#log_file =/var/log/qemu/qemu-guest-$pid. log
#rbd_concurrent_management_ops = 20
- Verify that the cache is in effect
- Validate effective command
[Email protected] guests]# ceph--admin-daemo ceph-client.cinder.27346.94699498717184.asok perf Dump
{
"Finisher-librbd::taskfinisher::m_finisher": {
"Queue_len": 0,
"Complete_latency": {
"Avgcount": 0,
"Sum": 0.000000000
}
},
"Librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
"RD": 32853,
"Rd_bytes": 2060131840,
"Rd_latency": {
"Avgcount": 32853,
"Sum": 42.348302430
},
"WR": 0,
"Wr_bytes": 0,
"Wr_latency": {
"Avgcount": 0,
"Sum": 0.000000000
},
"Discard": 0,
"Discard_bytes": 0,
"Discard_latency": {
"Avgcount": 0,
"Sum": 0.000000000
},
"Flush": 0,
"Aio_flush": 0,
"Aio_flush_latency": {
"Avgcount": 0,
"Sum": 0.000000000
},
"Snap_create": 0,
"Snap_remove": 0,
"Snap_rollback": 0,
"Snap_rename": 0,
"Notify": 0,
"Resize": 0,
"ReadAhead": 103,
"Readahead_bytes": 27451904,
"Invalidate_cache": 0
},
"Librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
"RD": 40116,
"Rd_bytes": 650777600,
"Rd_latency": {
"Avgcount": 40116,
"Sum": 38.547805260
},
"WR": 7348,
"Wr_bytes": 253340672,
"Wr_latency": {
"Avgcount": 7348,
"Sum": 55.895274365
},
"Discard": 0,
"Discard_bytes": 0,
"Discard_latency": {
"Avgcount": 0,
"Sum": 0.000000000
},
"Flush": 0,
"Aio_flush": 434,
"Aio_flush_latency": {
"Avgcount": 434,
"Sum": 0.540130077
},
"Snap_create": 0,
"Snap_remove": 0,
"Snap_rollback": 0,
"Snap_rename": 0,
"Notify": 0,
"Resize": 0,
"ReadAhead": 130,
"Readahead_bytes": 37353472,
"Invalidate_cache": 0
},
"Objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
"Cache_ops_hit": 2761,
"Cache_ops_miss": 30195,
"Cache_bytes_hit": 1500713472,
"Cache_bytes_miss": 2063611392,
"Data_read": 2087583744,
"Data_written": 0,
"data_flushed": 0,
"Data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"Write_time_blocked": 0.000000000
},
"Objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
"Cache_ops_hit": 829,
"Cache_ops_miss": 7979,
"Cache_bytes_hit": 191516672,
"Cache_bytes_miss": 153311232,
"Data_read": 168047616,
"Data_written": 253340672,
"Data_flushed": 253340672,
"Data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"Write_time_blocked": 0.000000000
},
"Objecter": {
"Op_active": 0,
"Op_laggy": 0,
"Op_send": 45229,
"Op_send_bytes": 0,
"Op_resend": 0,
"Op_ack": 45105,
"Op_commit": 7893,
"Op": 45229,
"Op_r": 37337,
"Op_w": 7892,
"OP_RMW": 0,
"Op_pg": 0,
"Osdop_stat": 7045,
"Osdop_create": 0,
"Osdop_read": 37319,
"Osdop_write": 7519,
"Osdop_writefull": 0,
"Osdop_append": 0,
"Osdop_zero": 0,
"Osdop_truncate": 0,
"Osdop_delete": 0,
"Osdop_mapext": 0,
"Osdop_sparse_read": 0,
"Osdop_clonerange": 0,
"Osdop_getxattr": 0,
"Osdop_setxattr": 0,
"Osdop_cmpxattr": 0,
"Osdop_rmxattr": 0,
"Osdop_resetxattrs": 0,
"Osdop_tmap_up": 0,
"Osdop_tmap_put": 0,
"Osdop_tmap_get": 0,
"Osdop_call": 1117,
"Osdop_watch": 122,
"Osdop_notify": 1,
"Osdop_src_cmpxattr": 0,
"Osdop_pgls": 0,
"Osdop_pgls_filter": 0,
"Osdop_other": 1,
"Linger_active": 1,
"Linger_send": 2,
"Linger_resend": 0,
"Linger_ping": 121,
"Poolop_active": 0,
"Poolop_send": 0,
"Poolop_resend": 0,
"Poolstat_active": 0,
"Poolstat_send": 0,
"Poolstat_resend": 0,
"Statfs_active": 0,
"Statfs_send": 0,
"Statfs_resend": 0,
"Command_active": 0,
"Command_send": 0,
"Command_resend": 0,
"Map_epoch": 0,
"Map_full": 0,
"Map_inc": 0,
"Osd_sessions": 27966,
"Osd_session_open": 236,
"Osd_session_close": 0,
"Osd_laggy": 1,
"OMAP_WR": 0,
"Omap_rd": 0,
"Omap_del": 0
},
"Throttle-msgr_dispatch_throttler-radosclient": {
"Val": 0,
"Max": 104857600,
"Get": 52511,
"Get_sum": 2152111179,
"Get_or_fail_fail": 0,
"Get_or_fail_success": 0,
"Take": 0,
"Take_sum": 0,
"Put": 52511,
"Put_sum": 2152111179,
"Wait": {
"Avgcount": 0,
"Sum": 0.000000000
}
},
"Throttle-objecter_bytes": {
"Val": 0,
"Max": 104857600,
"Get": 45108,
"Get_sum": 2419831296,
"Get_or_fail_fail": 0,
"Get_or_fail_success": 45108,
"Take": 0,
"Take_sum": 0,
"Put": 44725,
"Put_sum": 2419831296,
"Wait": {
"Avgcount": 0,
"Sum": 0.000000000
}
},
"Throttle-objecter_ops": {
"Val": 0,
"Max": 1024,
"Get": 45108,
"Get_sum": 45108,
"Get_or_fail_fail": 0,
"Get_or_fail_success": 45108,
"Take": 0,
"Take_sum": 0,
"Put": 45108,
"Put_sum": 45108,
"Wait": {
"Avgcount": 0,
"Sum": 0.000000000
}
}
} Full Configuration
Filtering the cache string
[[email protected] guests]# ceph--admin-daemo ceph-client.cinder.27346.94699498717184.asok perf Dump |gre P cache-a 5-b 5 "Snap_rename": 0, "notify": 0, "resize": 0, "ReadAhead": 103, "readahead_bytes": 27451904, "Invalidate_cache": 0}, "LIBRB D-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk ": {" rd ": 57149," Rd_bytes ": 1148467200," rd_latency ": {--" snap_rename ": 0," notify ": 0, "Resize": 0, "ReadAhead": "Readahead_bytes": 37353472, "Invali Date_cache ": 0}," Objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10 ": { "Cache_ops_hit": 2791, "Cache_ops_miss": 45597, "Cache_bytes_hit": 152639897 6, "cache_bytes_miss ": 3029461504," Data_read ": 3057748992," Data_written ": 0," Data_f Lushed ": 0," data_overwritten_while_flushing ": 0," write_ops_blocked ": 0," WR Ite_bytes_blocked ": 0," write_time_blocked ": 0.000000000}," objectcacher-librbd-bf1708496d08 8e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk ": {" Cache_ops_hit ": 886," cache _ops_miss ": 9957," Cache_bytes_hit ": 248139264," Cache_bytes_miss ": 225620480, "Data_read": 240729088, "Data_written": 462767104, "data_flushed": 462767104, "Data_overwritten_while_flushing": 0, "write_ops_blocked": 0,
- Verify the No Cache command
[Email protected] guests]# ceph--admin-daemo ceph-client.cinder.138869.94547912278016.asok perf dump |grep cache-a 5-b 5
"Snap_rename": 0,
"Notify": 0,
"Resize": 0,
"ReadAhead": 0,
"Readahead_bytes": 0,
"Invalidate_cache": 0
},
"Librbd-bee65365887c0b-nova_instances-5cf85c18-6725-4674-88e4-60408c97f487_disk": {
"RD": 62937,
"Rd_bytes": 1261381632,
"Rd_latency": { "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 0, "readahead_bytes": 0, "invalidate_cache": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 95327 }
Ceph Cache Open Validation is in effect
