分析统计MySQL general日志 找出查询次数最多的SQL

当我们需要优化MySQL查询时，第一想到的是开启慢日志，慢日志可以看到执行消耗超过一定时间的SQL语句和未使用索引的SQL。但如果我们想分析所有SQL查询的分布，即哪类SQL查询次数最多的时候，我们可以开启general log来统计。

 

开启general log

 

  1. mysql> show  variables like '%general%';

+——————+————————————-+
| Variable_name | Value |
+——————+————————————-+
| general_log | OFF |
| general_log_file | /usr/local/mysql/data/localhost.log |
+——————+————————————-+

  1. mysql> set global general_log = "ON";

 

analysis-general-log.py脚本

 

  1. #!/usr/bin/python
  2.  
  3. # sort and count mysql general log
  4. # Author: Jason
  5. # Url: www.centos.bz
  6. # Email: admin#centos.bz
  7. # Created: UTC 2015-02-15 17:51:53
  8.  
  9. import re
  10. import sys
  11. import os
  12.  
  13. if len(sys.argv) == 2:
  14.     logPath = sys.argv[1]
  15.     if not os.path.exists(logPath):
  16.         print ("file " + logPath + " does not exists.")
  17.         sys.exit(1)
  18. else:
  19.     print ("Usage: " + sys.argv[0] + " logPath")
  20.     sys.exit(1)
  21.  
  22. logFo = open(logPath)
  23. match = 0
  24.  
  25. for line in logFo:
  26.     line = re.sub(r"\n","",line)
  27.     if match == 0:
  28.         # match line begin with numbers
  29.         lineMatch = re.match(r"\s+[0-9]+\s+.*",line,flags=re.I)
  30.         if lineMatch:
  31.             lineTmp = lineMatch.group(0)
  32.             match = match + 1
  33.             continue
  34.  
  35.     elif match == 1:
  36.         # match line begin with numbers
  37.         lineMatch = re.match(r"\s+[0-9]+\s+.*",line,flags=re.I)
  38.         if lineMatch:
  39.             # match only query
  40.             lineMatchQuery = re.match(r".*Query\s+(.*)",lineTmp,flags=re.I)
  41.             if lineMatchQuery:
  42.                 lineTmp = lineMatchQuery.group(1)
  43.                 # remove extra space
  44.                 lineTmp = re.sub(r"\s+", " ",lineTmp)
  45.                 # replace values (value) to values (x)
  46.                 lineTmp = re.sub(r"values\s*\(.*?\)", "values (x)",lineTmp,flags=re.I)
  47.                 # replace filed = 'value' to filed = 'x'
  48.                 lineTmp = re.sub(r"(=|>|<|>=|<=)\s*('|\").*?\2","\\1 'x'",lineTmp)
  49.                 # replace filed = value to filed = x
  50.                 lineTmp = re.sub(r"(=|>|<|>=|<=)\s*[0-9]+","\\1 x",lineTmp)
  51.                 # replace like 'value' to like 'x'
  52.                 lineTmp = re.sub(r"like\s+('|\").*?\1","like 'x'",lineTmp,flags=re.I)
  53.                 # replace in (value) to in (x)
  54.                 lineTmp = re.sub(r"in\s+\(.*?\)","in (x)",lineTmp,flags=re.I)
  55.                 # replace limit x,y to limit
  56.                 lineTmp = re.sub(r"limit.*","limit",lineTmp,flags=re.I)
  57.                 
  58.                 print (lineTmp)
  59.  
  60.             match = 1
  61.             lineTmp = lineMatch.group(0)
  62.         else:   
  63.             lineTmp += line
  64.             match = 1
  65.  
  66. logFo.close()

使用方法：

  1. analysis-general-log.py general.log | sort | uniq -c | sort -nr


1032 SELECT * FROM wp_comments WHERE ( comment_approved = ‘x’ OR comment_approved = ‘x’ ) AND comment_post_ID = x ORDER BY comment_date_gmt DESC
653 SELECT post_id, meta_key, meta_value FROM wp_postmeta WHERE post_id in (x) ORDER BY meta_id ASC
527 SELECT FOUND_ROWS()
438 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy = ‘x’ AND t.term_id = x limit
341 SELECT option_value FROM wp_options WHERE option_name = ‘x’ limit
329 SELECT t.*, tt.*, tr.object_id FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON tt.term_id = t.term_id INNER JOIN wp_term_relationships AS tr ON tr.term_taxonomy_id = tt.term_taxonomy_id WHERE tt.taxonomy in (x) AND tr.object_id in (x) ORDER BY t.name ASC
311 SELECT wp_posts.* FROM wp_posts WHERE 1= x AND wp_posts.ID in (x) AND wp_posts.post_type = ‘x’ AND ((wp_posts.post_status = ‘x’)) ORDER BY wp_posts.post_date DESC
219 SELECT wp_posts.* FROM wp_posts WHERE ID in (x)
218 SELECT tr.object_id FROM wp_term_relationships AS tr INNER JOIN wp_term_taxonomy AS tt ON tr.term_taxonomy_id = tt.term_taxonomy_id WHERE tt.taxonomy in (x) AND tt.term_id in (x) ORDER BY tr.object_id ASC
217 SELECT wp_posts.* FROM wp_posts WHERE 1= x AND wp_posts.ID in (x) AND wp_posts.post_type = ‘x’ AND ((wp_posts.post_status = ‘x’)) ORDER BY wp_posts.menu_order ASC
202 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = ‘x’ AND (wp_posts.post_status = ‘x’) ORDER BY wp_posts.post_date DESC limit
118 SET NAMES utf8
115 SET SESSION sql_mode= ‘x’
115 SELECT @@SESSION.sql_mode
112 SELECT option_name, option_value FROM wp_options WHERE autoload = ‘x’
111 SELECT user_id, meta_key, meta_value FROM wp_usermeta WHERE user_id in (x) ORDER BY umeta_id ASC
108 SELECT YEAR(min(post_date_gmt)) AS firstdate, YEAR(max(post_date_gmt)) AS lastdate FROM wp_posts WHERE post_status = ‘x’
108 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy in (x) AND tt.count > x ORDER BY tt.count DESC limit
107 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy in (x) AND t.term_id in (x) ORDER BY t.name ASC
107 SELECT * FROM wp_users WHERE ID = ‘x’
106 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = ‘x’ AND (wp_posts.post_status = ‘x’) AND post_date > ‘x’ ORDER BY wp_posts.post_date DESC limit
106 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = ‘x’ AND (wp_posts.post_status = ‘x’) AND post_date > ‘x’ ORDER BY RAND() DESC limit
105 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = ‘x’ AND (wp_posts.post_status = ‘x’) AND post_date > ‘x’ ORDER BY wp_posts.comment_count DESC limit

标签：MySQL 发布于：2019-11-22 02:15:45
标签云
精彩推荐
猜你喜欢