作者:じ☆ve不哭
> 禁用win11 win10自动更新到2999年 ``` @echo off :menu cls echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo 请选择功能: echo 1. 暂停更新至2999年 echo 2. 恢复更新 echo 3. 彻底禁止更新(不可恢复) echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ set /p choice=请输入选项(1或2或3): if "%choice%"=="1" ( call :pause_updates ) else if "%choice%"=="2" ( call :resume_updates ) else if "%choice%"=="3" ( call :disable_updates ) else ( echo 无效的选项,请重新输入。 timeout /t 2 >nul goto menu ) pause exit :pause_updates echo 暂停更新... reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "FlightSettingsMaxPauseDays" /t REG_DWORD /d 7152 /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseFeatureUpdatesStartTime" /t REG_SZ /d "2024-01-01T10:00:52Z" /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseFeatureUpdatesEndTime" /t REG_SZ /d "2999-12-01T09:59:52Z" /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseQualityUpdatesStartTime" /t REG_SZ /d "2024-01-01T10:00:52Z" /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseQualityUpdatesEndTime" /t REG_SZ /d "2999-12-01T09:59:52Z" /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseUpdatesStartTime" /t REG_SZ /d "2024-01-01T09:59:52Z" /f reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseUpdatesExpiryTime" /t REG_SZ /d "2999-12-01T09:59:52Z" /f echo 更新已暂停。 timeout /t 2 >nul goto :eof :resume_updates echo 恢复默认... reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "FlightSettingsMaxPauseDays" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseFeatureUpdatesStartTime" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseFeatureUpdatesEndTime" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseQualityUpdatesStartTime" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseQualityUpdatesEndTime" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseUpdatesStartTime" /f reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" /v "PauseUpdatesExpiryTime" /f echo 已恢复默认设置。 timeout /t 2 >nul goto :eof :disable_updates echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 结束进程 / Stopping Process... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ taskkill /im Windows10UpgraderApp.exe 2>nul del /f /q "%USERPROFILE%\Desktop\微软 Windows 10 易升.lnk" 2>nul del /f /q "%USERPROFILE%\Desktop\Windows 10 Update Assistant.lnk" 2>nul echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 添加防火墙规则 / Adding firewall rules... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ netsh advfirewall firewall add rule name="Block_Windows10UpgraderApp" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\Windows10UpgraderApp.exe" action=block netsh advfirewall firewall add rule name="Block_WinREBootApp32" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\WinREBootApp32.exe" action=block netsh advfirewall firewall add rule name="Block_WinREBootApp64" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\WinREBootApp64.exe" action=block netsh advfirewall firewall add rule name="Block_bootsect" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\bootsect.exe" action=block netsh advfirewall firewall add rule name="Block_DW20" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\DW20.EXE" action=block netsh advfirewall firewall add rule name="Block_DWTRIG20" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\DWTRIG20.EXE" action=block netsh advfirewall firewall add rule name="Block_GatherOSState" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\GatherOSState.EXE" action=block netsh advfirewall firewall add rule name="Block_GetCurrentRollback" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\GetCurrentRollback.EXE" action=block netsh advfirewall firewall add rule name="Block_HttpHelper" dir=in program="%SYSTEMDRIVE%\Windows10Upgrade\HttpHelper.exe" action=block netsh advfirewall firewall add rule name="Block_UpdateAssistant" dir=in program="%SYSTEMROOT%\UpdateAssistant\UpdateAssistant.exe" action=block netsh advfirewall firewall add rule name="Block_UpdateAssistantCheck" dir=in program="%SYSTEMROOT%\UpdateAssistant\UpdateAssistantCheck.exe" action=block netsh advfirewall firewall add rule name="Block_Windows10Upgrade" dir=in program="%SYSTEMROOT%\UpdateAssistant\Windows10Upgrade.exe" action=block netsh advfirewall firewall add rule name="Block_UpdateAssistantV2" dir=in program="%SYSTEMROOT%\UpdateAssistantV2\UpdateAssistant.exe" action=block netsh advfirewall firewall add rule name="Block_UpdateAssistantCheckV2" dir=in program="%SYSTEMROOT%\UpdateAssistantV2\UpdateAssistantCheck.exe" action=block netsh advfirewall firewall add rule name="Block_Windows10UpgradeV2" dir=in program="%SYSTEMROOT%\UpdateAssistantV2\Windows10Upgrade.exe" action=block echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 设置ACL / Configurating ACL... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo y|cacls C:\Windows\UpdateAssistant\*.exe /t /p everyone:n 2>nul echo y|cacls C:\Windows10Upgrade\*.exe /t /p everyone:n 2>nul echo. echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 停止Windows Update服务 / Disable Windows Update ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ net stop wuauserv sc config wuauserv start= disabled echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 删除计划任务 / Delete task... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ schtasks /delete /TN "\Microsoft\Windows\UpdateOrchestrator\UpdateAssistant" /f 2>nul schtasks /delete /TN "\Microsoft\Windows\UpdateOrchestrator\UpdateAssistantAllUsersRun" /f 2>nul schtasks /delete /TN "\Microsoft\Windows\UpdateOrchestrator\UpdateAssistantCalendarRun" /f 2>nul schtasks /delete /TN "\Microsoft\Windows\UpdateOrchestrator\UpdateAssistantWakeupRun" /f 2>nul echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 设置注册表 / Editing Registry... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ del /q /f %SYSTEMDRIVE%\NAU.reg 2>nul echo Windows Registry Editor Version 5.00 >> %SYSTEMDRIVE%\NAU.reg echo.>> %SYSTEMDRIVE%\NAU.reg echo [HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate]>> %SYSTEMDRIVE%\NAU.reg echo "DoNotConnectToWindowsUpdateInternetLocations"=dword:00000001 >> %SYSTEMDRIVE%\NAU.reg echo.>> %SYSTEMDRIVE%\NAU.reg echo [HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU]>> %SYSTEMDRIVE%\NAU.reg echo "NoAutoUpdate"=dword:00000001>> %SYSTEMDRIVE%\NAU.reg REG IMPORT %SYSTEMDRIVE%\NAU.reg del /q /f %SYSTEMDRIVE%\NAU.reg 2>nul echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ echo ※ ※ echo ※ 更新已禁用 / Updates are disabled... ※ echo ※ ※ echo ※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※※ pause goto :eof ``` ## 成品下载地址 解压密码:studyjava [disabled_update.7z](https://studyjava.cn/files/articleVideo/zsljava11758247578827463.7z)
作者:じ☆ve不哭
## 依赖 - Java:安装 Java 17 或更高版本(Kafka 4.0+ 要求) ## 下载kafka ``` wget https://downloads.apache.org/kafka/4.0.0/kafka_2.13-4.0.0.tgz tar -xzf kafka_2.13-4.0.0.tgz cd kafka_2.13-4.0.0 ``` ## 开始安装 **1、生成集群唯一标识符(Cluster ID)** Kafka 4.0 完全移除了对 ZooKeeper 的依赖,因此我们只需要配置 KRaft。 ``` ./bin/kafka-storage.sh random-uuid # 输出类似:cELBcqLNQAuEvTfQgdX4_A # 复制这个 UUID,下一步会用到。 ``` **2、修改config/server.properties** 最终配置如下 ``` broker.id=1 process.roles=broker,controller node.id=1 controller.quorum.voters=1@localhost:9093 listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 inter.broker.listener.name=PLAINTEXT advertised.listeners=PLAINTEXT://123.56.190.57:9092 controller.listener.names=CONTROLLER listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=/tmp/kraft-combined-logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 share.coordinator.state.topic.replication.factor=1 share.coordinator.state.topic.min.isr=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 ``` **3、使用上一步生成的 UUID 来初始化存储元数据的日志目录。** ``` ./bin/kafka-storage.sh format -t <uuid> -c ./config/server.properties # 例如: ./bin/kafka-storage.sh format -t QDK7gwhnQT-w_iputukn6w -c ./config/server.properties ``` **4、启动** ``` # 后台启动 bin/kafka-server-start.sh -daemon config/server.properties # 或者前台启动(查看日志) bin/kafka-server-start.sh config/server.properties ``` ## 验证 **1. 创建一个 Topic** 打开另一个终端,执行: ``` ./bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1 ``` **2. 生产一些消息** ``` ./bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 > Hello Kafka 4.0! > This is KRaft mode. ``` **3. 消费这些消息** ``` ./bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 ```
作者:じ☆ve不哭
 # Ubuntu挂载硬盘 ``` // 1. 检查硬盘,找到要挂载的硬盘 fdisk -l // 2. 创建挂载点 mkdir /data // 3.1 没使用过的先格式化硬盘 mkfs -t ext4 /dev/sdb // 3.2 挂载硬盘 mount /dev/sdb /data // 4.1 启用开机自动挂载 nano /etc/fstab // 4.2 在文件结尾添加挂载信息 /dev/sdb /data ext4 defaults 0 0 ```
作者:微信小助手
<section style="font-size: 15px;box-sizing: border-box;font-style: normal;font-weight: 400;text-align: justify;color: rgb(62, 62, 62);" data-pm-slice="0 0 []"> <section style="margin: 10px 0% 8px;text-align: left;justify-content: flex-start;display: flex;flex-flow: row;width: 100%;border-left: 3px solid rgb(219, 219, 219);border-bottom-left-radius: 0px;padding: 0px 0px 0px 8px;align-self: flex-start;box-sizing: border-box;"> <section style="color: rgba(0, 0, 0, 0.5);font-size: 14px;text-align: justify;width: 100%;box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">作者:vivo 互联网存储团队 - Lin Haiwen、Xu Xingbao</span></p> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="margin: 10px 0%;text-align: left;justify-content: flex-start;display: flex;flex-flow: row;box-sizing: border-box;"> <section style="display: inline-block;width: 100%;vertical-align: top;border-bottom: 1px dashed rgb(65, 95, 255);border-bottom-right-radius: 0px;border-right: 1px dashed rgb(65, 95, 255);border-top-right-radius: 0px;border-left-width: 0px;align-self: flex-start;flex: 0 0 auto;box-sizing: border-box;"> <section style="margin: 0px 0% 10px;box-sizing: border-box;"> <section style="display: inline-block;width: 96%;border-style: solid;border-width: 1px 0px 0px 10px;padding: 0px 10px;box-shadow: rgb(0, 0, 0) 0px 0px 0px;border-color: rgb(65, 95, 255);box-sizing: border-box;"> <section style="margin: 10px 0% 0px;box-sizing: border-box;"> <section style="color: rgb(65, 95, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">目录</span></strong></p> </section> </section> </section> </section> <section style="transform: translate3d(20px, 0px, 0px);-webkit-transform: translate3d(20px, 0px, 0px);-moz-transform: translate3d(20px, 0px, 0px);-o-transform: translate3d(20px, 0px, 0px);box-sizing: border-box;"> <section style="text-align: justify;color: rgb(65, 95, 255);padding: 0px;box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">01. 问题描述</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">02. 问题定位</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">03. 问题解决</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">04. 总 结</span></p> </section> </section> <section style="transform: scale(0.9);-webkit-transform: scale(0.9);-moz-transform: scale(0.9);-o-transform: scale(0.9);transform-origin: center center;-webkit-transform-origin: center center;-moz-transform-origin: center center;-o-transform-origin: center center;margin-top: 0px;margin-bottom: 0px;box-sizing: border-box;"> <section style="display: flex;flex-flow: row;margin: 10px 0%;justify-content: flex-start;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: middle;width: auto;flex: 100 100 0%;align-self: center;height: auto;border-width: 0px;box-sizing: border-box;"> <section style="margin: 0px 0%;box-sizing: border-box;"> <section style="background-color: rgb(226, 226, 226);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: middle;width: auto;align-self: center;min-width: 10%;max-width: 100%;flex: 0 0 auto;line-height: 0;letter-spacing: 0px;height: auto;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="display: inline-block;width: 5px;height: 5px;vertical-align: top;overflow: hidden;border-width: 0px;border-radius: 50%;border-style: none;border-color: rgb(62, 62, 62);background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: middle;width: auto;align-self: center;flex: 100 100 0%;box-sizing: border-box;"> <section style="margin: 0px 0%;box-sizing: border-box;"> <section style="background-color: rgb(226, 226, 226);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> </section> <section style="text-align: justify;justify-content: flex-start;display: flex;flex-flow: row;box-sizing: border-box;"> <section style="display: inline-block;width: 100%;vertical-align: top;align-self: flex-start;flex: 0 0 auto;padding: 5px 20px;box-sizing: border-box;"> <section style="margin: 8px 0%;box-sizing: border-box;"> <section style="color: rgb(62, 62, 62);text-align: left;padding: 0px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">本文从一次生产环境业务服务报错,逐步对问题进行定位,深入分析之后发现导致问题的原因,给出相应的优化方法,提升业务可用性。</span></p> </section> </section> </section> </section> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1分钟看图掌握核心观点👇</span></p> </section> <section style="text-align: center;margin: 10px 0% 20px;justify-content: center;display: flex;flex-flow: row;box-sizing: border-box;"> <section style="display: inline-block;width: 97%;vertical-align: top;box-shadow: rgb(140, 140, 140) 0px 0px 3px;align-self: flex-start;flex: 0 0 auto;box-sizing: border-box;"> <section style="margin: 6px 0% 0px;line-height: 0;box-sizing: border-box;"> <section style="max-width: 100%;vertical-align: middle;display: inline-block;line-height: 0;width: 96%;box-shadow: rgb(0, 0, 0) 0px 0px 0px;box-sizing: border-box;" nodeleaf=""> <img src="https://mmecoa.qpic.cn/sz_mmecoa_gif/4g5IMGibSxt44zrScVX9tJNkEvqHfGqficAjEGOy4KyykQibCs9TAc4OKfk7bUb94lnT8QFcMyZfuXZbLNTo7VEGw/640?wx_fmt=gif&from=appmsg" class="rich_pages wxw-img" data-ratio="0.996875" data-type="gif" data-w="640" style="vertical-align: middle;max-width: 100%;width: 100%;box-sizing: border-box;" src="/upload/aab2db9ea251cec3779c2a6821c666b6.png" data-cropselx2="538" data-cropsely2="359" data-imgfileid="100021978"> </section> </section> <section style="margin: 0px 0% -22px;line-height: 0;box-sizing: border-box;"> <section style="max-width: 100%;vertical-align: middle;display: inline-block;line-height: 0;box-shadow: rgb(0, 0, 0) 0px 0px 0px;box-sizing: border-box;" nodeleaf=""> <img src="/upload/6bfbfa93f623b7478ed0ae31d8147ff1.png" class="rich_pages wxw-img" data-ratio="0.078125" data-s="300,640" data-type="png" data-w="640" style="vertical-align: middle;max-width: 100%;width: 100%;box-sizing: border-box;" data-imgfileid="100021903"> </section> </section> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="display: flex;flex-flow: row;margin: 10px 0px;text-align: center;justify-content: center;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;border-bottom: 0px solid rgb(169, 207, 245);border-bottom-right-radius: 0px;flex: 0 0 auto;align-self: flex-end;min-width: 10%;max-width: 100%;height: auto;padding: 0px 12px;margin: 0px;box-sizing: border-box;"> <section style="font-size: 20px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><em style="box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">01</span></strong></em></p> </section> <section style="margin: 2px 0px 0px;box-sizing: border-box;"> <section style="background-color: rgb(65, 95, 255);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> </section> <section style="text-align: center;font-size: 18px;color: rgb(65, 95, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">问题描述</span></p> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1.1</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">报错信息</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">应用服务报错,通过监控日志发现凌晨2点的时候,应用报错获取不到Redis key。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1.2</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">告警与监控信息</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">首先想到是否由于内存满导致的key淘汰,生产的所有Redis都有设置内存告警,但没有收到内存告警信息;(内存告警需要每隔10秒,连续3次触发才会告警。)</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">从监控图中,可以看到Redis内存稍有增长,但使用率一直偏低,并没有达到使用率告警。</span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/e8094ea6b7831c790559a9178ddb5530.png" class="rich_pages wxw-img" data-ratio="0.828125" data-s="300,640" data-type="png" data-w="2304" type="block" data-imgfileid="100021909"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">查看监控信息:在问题时间点,发生了大量的key过期,初步怀疑是由于key批量设置了过期时间,正好到期了导致大量key失效。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/5338bf7824ef3e825ed7544e87364a85.png" class="rich_pages wxw-img" data-ratio="0.73984375" data-s="300,640" data-type="webp" data-w="1280" type="block" data-imgfileid="100021907"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">查看Redis错误日志,没有发现异常。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="display: flex;flex-flow: row;margin: 10px 0px;text-align: center;justify-content: center;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;border-bottom: 0px solid rgb(169, 207, 245);border-bottom-right-radius: 0px;flex: 0 0 auto;align-self: flex-end;min-width: 10%;max-width: 100%;height: auto;padding: 0px 12px;margin: 0px;box-sizing: border-box;"> <section style="font-size: 20px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><em style="box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">02</span></strong></em></p> </section> <section style="margin: 2px 0px 0px;box-sizing: border-box;"> <section style="background-color: rgb(65, 95, 255);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> </section> <section style="text-align: center;font-size: 18px;color: rgb(65, 95, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">问题定位</span></p> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="margin-bottom: -2.25em;margin-right: 5px;background-color: rgb(247, 247, 247);box-sizing: border-box;"> <section style="padding: 10px;margin-bottom: 5px;box-sizing: border-box;"> <section style="text-align: left;box-sizing: border-box;"> <ul style="list-style-type: disc;" class="list-paddingleft-1"> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">基于前面的监控,初步怀疑是key设置了过期时间导致失效。</span></p></li> </ul> <ul style="list-style-type: disc;" class="list-paddingleft-1"> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">是否有上线其他新功能导致?</span></p></li> </ul> </section> </section> </section> <section style="margin-left: auto;width: 2.25em;height: 2.25em;border-right: 5px solid transparent;border-bottom: 5px solid transparent;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">但是业务反馈不是由于设置过期时间导致;并且第二天问题复现,因此继续深入定位。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">2.1</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">key是否过期</span></strong></p> </section> </section> </section> </section> <section style="margin-top: 10px;margin-bottom: 10px;box-sizing: border-box;"> <section style="margin-bottom: -2.25em;margin-right: 5px;background-color: rgb(247, 247, 247);box-sizing: border-box;"> <section style="padding: 10px;margin-bottom: 5px;box-sizing: border-box;"> <section style="text-align: left;box-sizing: border-box;"> <ul style="list-style-type: disc;" class="list-paddingleft-1"> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">查看淘汰策略</span></p></li> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">查看key过期时间</span></p></li> </ul> </section> </section> </section> <section style="margin-left: auto;width: 2.25em;height: 2.25em;border-right: 5px solid transparent;border-bottom: 5px solid transparent;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">初步判断确实不是因为key过期导致的大量淘汰,这里TTL接近5天,但是连着2天出现问题,因此不应该是过期时间到了导致。</span></p> <section class="code-snippet__fix code-snippet__js"> <ul class="code-snippet__line-index code-snippet__js"> </ul> <pre class="code-snippet__js" data-lang="sql"><code><span leaf="">xxx:xxx<span class="code-snippet__operator">></span> config <span class="code-snippet__keyword">get</span><span class="code-snippet__string">'maxmemory-policy'</span></span></code><code><span leaf=""><span class="code-snippet__number">1</span>)"maxmemory-policy"</span></code><code><span leaf=""><span class="code-snippet__number">2</span>)"volatile-lru"</span></code><code><span leaf="">xxx:xxx<span class="code-snippet__operator">></span> ttl finance:xxxx_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__operator">-></span> Redirected <span class="code-snippet__keyword">to</span> slot [<span class="code-snippet__number">9229</span>] located <span class="code-snippet__keyword">at</span> xxx:xxx</span></code><code><span leaf="">(<span class="code-snippet__type">integer</span>)<span class="code-snippet__number">387585</span></span></code><code><span leaf="">xxx:xxx<span class="code-snippet__operator">></span> ttl finance:xxxcms_basic_data_10423</span></code><code><span leaf="">(<span class="code-snippet__type">integer</span>)<span class="code-snippet__number">387574</span></span></code></pre> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">key是被删除还是淘汰?查看监控,发现存在key确实被淘汰,说明接下来需要考虑内存问题。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/e374b63f35e36aed988c5d94dc4d659b.png" class="rich_pages wxw-img" data-ratio="0.5836298932384342" data-s="300,640" data-type="png" data-w="562" type="block" data-imgfileid="100021906"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">2.2</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">是否内存满了</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">发现确实短时间内存写满了,一开始查看监控由于时间拉的比较长,查看增长趋势没发现内存写满情况,但是由于没有达到告警条件,没有满足连着3次触发阈值,故没有触发告警。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">同时内存用满问题持续时间较短,约10分钟左右。</span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/94faa30b682476eb0b23995d33a37273.png" class="rich_pages wxw-img" data-ratio="0.6503856041131105" data-s="300,640" data-type="webp" data-w="778" type="block" data-imgfileid="100021908"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">其他指标检查,发现出现问题时client_longest_output_list指标存在明显突刺,该指标非常可疑。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/7564088bb7bb85dd40d6c9643a7dfe4f.png" class="rich_pages wxw-img" data-ratio="0.4390625" data-s="300,640" data-type="webp" data-w="1280" type="block" data-imgfileid="100021911"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">请求量的大涨,怀疑是请求堆积导致buffer增长使得内存写满。但是此时还有疑点:</span><em style="box-sizing: border-box;"><span leaf="">写入也上涨,是否是由于读请求积压导致,还是因为写入数据导致内存满?</span></em></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">2.3</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">找出内存涨的来源</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">设置定时任务,对出问题时间点前后20分钟这段时间进行抓包分析。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">对比出问题前后几分钟的请求,对应时间点请求量飙升,并且请求量来源基本是get请求,set请求也少量增长。</span></p> <section class="code-snippet__fix code-snippet__js"> <ul class="code-snippet__line-index code-snippet__js"> </ul> <pre class="code-snippet__js" data-lang="sql"><code><span leaf=""><span class="code-snippet__number">4860</span> <span class="code-snippet__keyword">get</span> finance:xxxx__10122</span></code><code><span leaf=""><span class="code-snippet__number">4925</span> <span class="code-snippet__keyword">get</span> finance:xxxx__10032</span></code><code><span leaf=""><span class="code-snippet__number">4945</span><span class="code-snippet__keyword">set</span> finance:xxxx_data_10013<span class="code-snippet__number">-0</span></span></code><code><span leaf=""><span class="code-snippet__number">4947</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10013_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">4976</span> <span class="code-snippet__keyword">get</span> finance:xxxx__10251</span></code><code><span leaf=""><span class="code-snippet__number">5054</span><span class="code-snippet__keyword">set</span> finance:xxxx__undefined</span></code><code><span leaf=""><span class="code-snippet__number">5098</span> <span class="code-snippet__keyword">get</span> finance:xxxx__10018</span></code><code><span leaf=""><span class="code-snippet__number">8729</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10415_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9152</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10401_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9553</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10228_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9597</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10213_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9622</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10032_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9647</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10347_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9674</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10182_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">9742</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10251_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">10085</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10019_cms_version_10000</span></code><code><span leaf=""><span class="code-snippet__number">23064</span> <span class="code-snippet__keyword">get</span> finance:xxxx__10423</span></code><code><span leaf=""><span class="code-snippet__number">45176</span> <span class="code-snippet__keyword">get</span> finance:xxxx_data_10423_cms_version_10000 </span></code><code><span leaf=""><br></span></code><code><span leaf="">[root<span class="code-snippet__variable">@db</span><span class="code-snippet__operator">-</span>prd<span class="code-snippet__operator">-</span>xxx.v<span class="code-snippet__operator">-</span>bj<span class="code-snippet__number">-3.</span>vivo.lan:<span class="code-snippet__operator">/</span>data<span class="code-snippet__operator">/</span>redis<span class="code-snippet__operator">/</span>scpdir]</span></code><code><span leaf=""><br></span></code><code><span leaf=""># cat <span class="code-snippet__number">0807.</span>cap <span class="code-snippet__operator">|</span> grep <span class="code-snippet__string">'2024-08-07 01:59'</span><span class="code-snippet__operator">|</span>awk <span class="code-snippet__string">'{print $8,$10}'</span><span class="code-snippet__operator">|</span>sort <span class="code-snippet__operator">|</span>uniq <span class="code-snippet__operator">-</span>c <span class="code-snippet__operator">|</span>sort <span class="code-snippet__operator">-</span>k <span class="code-snippet__number">1</span> <span class="code-snippet__operator">-</span>n</span></code></pre> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">同时也对set的内容进行分析,发现set的数据并不足使内存写满;且上面监控可以看到,内存写满问题持续时间很短,因此应该不是数据增长导致。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">进一步对get请求来源分析:</span></p> </section> <section style="min-height: 40px;margin: 10px 0%;box-sizing: border-box;"> <section style="width: 100%;margin: 0px auto -10px;box-sizing: border-box;"> <table style="border-collapse: collapse;box-sizing: border-box;margin-bottom: 10px;"> <tbody> <tr style="box-sizing: border-box;"> <td data-colwidth="40.0000%" width="40.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">次数</span></strong></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">IP来源</span></strong></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="40.0000%" width="40.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">161179</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1xx.xx.xx.72</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="40.0000%" width="40.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">138236</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1xx.xx.xx.215</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="40.0000%" width="40.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">130110</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1xx.xx.xx.190</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="40.0000%" width="40.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">101877</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">1xx.xx.xx.185</span></p> </section> </section></td> </tr> </tbody> </table> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">结合 IP来源以及 keyname;跟业务同学沟通确认:</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">由于业务读请求量大涨导致,业务请求从每分钟27w左右上涨到70w左右,主要有:</span></p> </section> <section style="min-height: 40px;margin: 10px 0%;box-sizing: border-box;"> <section style="width: 100%;margin: 0px auto -10px;box-sizing: border-box;"> <table style="border-collapse:collapse;box-sizing:border-box;margin-bottom:10px;min-width:151px;"> <tbody> <tr style="box-sizing: border-box;"> <td data-colwidth="20.0000%" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">次数</span></strong></p> </section> </section></td> <td data-colwidth="101" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">IP来源</span></strong></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: center;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">keyname</span></strong></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="20.0000%" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">330660</span></p> </section> </section></td> <td data-colwidth="101" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">get</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: left;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">finance:f2e:xxx_cms_version_10000</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="20.0000%" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">230735</span></p> </section> </section></td> <td data-colwidth="101" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">get</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: left;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">finance:f2e:xxx_data_10423</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="20.0000%" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">36653</span></p> </section> </section></td> <td data-colwidth="101" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">set</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: left;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">finance:f2e:xxx_data_10423</span></p> </section> </section></td> </tr> <tr style="box-sizing: border-box;"> <td data-colwidth="20.0000%" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">34251</span></p> </section> </section></td> <td data-colwidth="101" width="20.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="padding: 0px 5px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">set</span></p> </section> </section></td> <td data-colwidth="60.0000%" width="60.0000%" style="border-width: 1px;border-color: rgb(62, 62, 62);border-style: solid;box-sizing: border-box;padding: 0px;"> <section style="margin: 5px 0%;box-sizing: border-box;"> <section style="text-align: left;padding: 0px 5px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">finance:f2e:xxx_cms_version_10000</span></p> </section> </section></td> </tr> </tbody> </table> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">2.4</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">机制分析</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">内存用量上涨超限会触发Redis节点基于已经配置的volatile-lru策略进行过期数据淘汰,所以需要找到内存上涨的来源。基于监控指标排查分析,基本确定了发生异常期间没有集中的大量数据写入,反而发现大量网络数据的输出,抓包也印证了此时节点主要是在处理get命令读请求。进一步地发现了client-output-buffer-limit这个指标出现异常上涨,才最终锁定到Redis的回包积压问题。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">关于Redis的结果回包逻辑,首先要了解Redis的结果存储空间设计。Redis针对每一个连接客户端都会初始化一个大小为16K的静态的buf区域和一个空的链表结果,相关声明代码如下:</span></p> <section class="code-snippet__fix code-snippet__js"> <ul class="code-snippet__line-index code-snippet__js"> </ul> <pre class="code-snippet__js" data-lang="cpp"><code><span leaf=""><span class="code-snippet__meta">#</span><span class="code-snippet__meta"><span class="code-snippet__keyword">define</span></span><span class="code-snippet__meta"> REDIS_REPLY_CHUNK_BYTES (16*1024) </span><span class="code-snippet__meta"><span class="code-snippet__comment">/* 16k output buffer */</span></span></span></code><code><span leaf=""><span class="code-snippet__type">char</span> buf[REDIS_REPLY_CHUNK_BYTES];</span></code><code><span leaf="">list *reply;</span></code><code><span leaf="">c->reply = <span class="code-snippet__built_in">listCreate</span>();</span></code></pre> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">对于Redis而言正常执行的命令都会有数据回包,而回包结果都需要在客户端中做暂存。Redis是如何结合以上两个数据进行结果存储的呢?主要逻辑是如果静态buf区域能够满足回包结果存储,即结果不大于16k,那么结果就会存储静态buf中,将结果不断插入reply链表中;同时我们都知道Redis支持丰富的数据类型和操作命令,有些批量数据读取命令的结果可能会有很多字段,这些字段也会作为一个个链表元素追加到reply链表中。正常情况下,我们在Redis节点上执行info clients命令可以获得如下客户端的统计信息:</span></p> <section class="code-snippet__fix code-snippet__js"> <ul class="code-snippet__line-index code-snippet__js"> </ul> <pre class="code-snippet__js" data-lang="shell"><code><span leaf=""><span class="code-snippet__meta"> ></span> info clients </span></code><code><span leaf=""><span class="code-snippet__meta"> #</span> Clients </span></code><code><span leaf=""> connected_clients: 1</span></code><code><span leaf=""> client_longest_output_list: 0</span></code><code><span leaf=""> client_biggest_input_buf: 0</span></code><code><span leaf=""> blocked_clients: </span></code></pre> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">其中的client_longest_output_list字段代表此时节点的所有连接客户端中回包结果的缓存情况。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">但是按照之前服务监控和抓包结果分析,具体执行的指令都是get,实际数据也没有超过16k大小,并没有满足将结果存储到链表的条件。这里有个经常被忽略的场景,就是Redis其实支持pipeline命令执行方式的,简单来说就是Redis支持一次性接收一个客户端的多个指令,具体执行过程中会把这些指令的结果不断暂存,然后在后续流程中集中回包,如果这时候不能及时地把数据通过网络发出去,就有可能出现reply链表长度激增的现象,进而导致客户端占用内存激增,这正是我们本次遇到的场景。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;" nodeleaf=""> <img src="/upload/ae6a23e3711fca8ffa76e4df2234f9f7.png" class="rich_pages wxw-img" data-ratio="0.678125" data-s="300,640" data-type="webp" data-w="1280" type="block" data-imgfileid="100021910"> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">Redis的maxmemory参数限制的是Redis实例可以使用的最大内存,这部分内存主要包括以下几个部分:业务数据占用的内存、客户端连接的输入/输出缓冲区、复制积压缓冲区、AOF 缓冲区以及其他一些内部开销。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">具体来说,Redis 的maxmemory 限制包括:</span></p> </section> <section style="margin-top: 10px;margin-bottom: 10px;box-sizing: border-box;"> <section style="margin-bottom: -2.25em;margin-right: 5px;background-color: rgb(247, 247, 247);box-sizing: border-box;"> <section style="padding: 10px;margin-bottom: 5px;box-sizing: border-box;"> <section style="text-align: left;box-sizing: border-box;"> <ol style="list-style-type: decimal;" class="list-paddingleft-1"> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">业务数据占用的内存,这部分是用户在Redis中存储的数据。</span></p></li> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">客户端连接的输入/输出缓冲区,用于暂存客户端发送的命令和Redis 返回给客户端的数据。</span></p></li> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">复制积压缓冲区:用于主从复制,当从节点断线重连时,可以从这个缓冲区拉取丢失的数据。</span></p></li> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">AOF 缓冲区:用于持久化,当开启AOF 持久化时,Redis 会将写操作追加到AOF 缓冲区,然后异步地写入AOF 文件。</span></p></li> <li><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">其他内部开销:包括Redis 进程本身的一些数据结构、对象、碎片内存等。</span></p></li> </ol> </section> </section> </section> <section style="margin-left: auto;width: 2.25em;height: 2.25em;border-right: 5px solid transparent;border-bottom: 5px solid transparent;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">因此,在设置Redis 的maxmemory 参数时,需要综合考虑业务数据的内存占用、各个缓冲区的大小以及内存碎片率等因素,合理地分配内存,避免出现内存溢出或性能下降的问题。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="display: flex;flex-flow: row;margin: 10px 0px;text-align: center;justify-content: center;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;border-bottom: 0px solid rgb(169, 207, 245);border-bottom-right-radius: 0px;flex: 0 0 auto;align-self: flex-end;min-width: 10%;max-width: 100%;height: auto;padding: 0px 12px;margin: 0px;box-sizing: border-box;"> <section style="font-size: 20px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><em style="box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">03</span></strong></em></p> </section> <section style="margin: 2px 0px 0px;box-sizing: border-box;"> <section style="background-color: rgb(65, 95, 255);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> </section> <section style="text-align: center;font-size: 18px;color: rgb(65, 95, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">问题解决</span></p> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">3.1</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">紧急修复</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="box-sizing: border-box;"> <ul style="list-style-type: disc;" class="list-paddingleft-1"> <li><p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">临时扩大Redis集群内存,避免内存写满。</span></p></li> <li><p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">限制client-output-buffer-limit大小,避免由于请求过大导致内存突增。</span></p></li> <li><p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">业务限流&削峰,避免请求量突增。</span></p></li> <li><p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">增加兜底机制,如果由于Redis key被淘汰,则去MySQL查询,避免业务直接报错。</span></p></li> </ul> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> </section> <section style="text-align: left;justify-content: flex-start;display: flex;flex-flow: row;margin: 10px 0px;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;background-color: rgb(65, 95, 255);min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 6px;border-top-left-radius: 6px;border-top-right-radius: 6px;border-bottom-left-radius: 6px;overflow: hidden;box-sizing: border-box;"> <section style="text-align: center;box-sizing: border-box;"> <section style="color: rgb(255, 255, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">3.2</span></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;line-height: 0;align-self: flex-end;min-width: 5%;max-width: 100%;flex: 0 0 auto;height: auto;padding: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="text-align: center;margin: 3px 0px;box-sizing: border-box;"> <section style="text-align: justify;line-height: 1;padding: 0px 4px;box-sizing: border-box;"> <p style="text-align: center;white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">根本解决</span></strong></p> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;align-self: flex-end;flex: 0 0 0%;height: auto;padding: 0px 2px 0px 0px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">业务进行业务逻辑优化,将请求打散,避免同一时间集中大量请求Redis。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="display: flex;flex-flow: row;margin: 10px 0px;text-align: center;justify-content: center;box-sizing: border-box;"> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;border-bottom: 0px solid rgb(169, 207, 245);border-bottom-right-radius: 0px;flex: 0 0 auto;align-self: flex-end;min-width: 10%;max-width: 100%;height: auto;padding: 0px 12px;margin: 0px;box-sizing: border-box;"> <section style="font-size: 20px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><em style="box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">04</span></strong></em></p> </section> <section style="margin: 2px 0px 0px;box-sizing: border-box;"> <section style="background-color: rgb(65, 95, 255);height: 1px;box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;margin: 0px 4px 0px 0px;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 16px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <section style="display: inline-block;vertical-align: bottom;width: auto;flex: 0 0 0%;height: auto;align-self: flex-end;box-sizing: border-box;"> <section style="transform: perspective(0px);-webkit-transform: perspective(0px);-moz-transform: perspective(0px);-o-transform: perspective(0px);transform-style: flat;box-sizing: border-box;"> <section style="transform: rotateX(180deg);-webkit-transform: rotateX(180deg);-moz-transform: rotateX(180deg);-o-transform: rotateX(180deg);margin: 0px;box-sizing: border-box;"> <section style="display: inline-block;width: 3px;height: 10px;vertical-align: top;overflow: hidden;background-color: rgb(65, 95, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> </section> <section style="text-align: center;font-size: 18px;color: rgb(65, 95, 255);box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">总 结</span></p> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="box-sizing: border-box;"> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">本次问题是由于业务集中请求Redis,导致缓存积压内存增长达到最大内存限制,触发Redis淘汰策略对key进行驱逐。key被淘汰丢失后,需要增加兜底机制去DB侧请求避免业务报错。虽然Redis性能比较好,但是也要尽量打散请求,合理评估存储侧的性能。</span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">同时,对于Redis淘汰策略,对于数据比较重要的集群,可以考虑使用不驱逐的方式。合理设置TTL保留时间,把Redis的内存使用率保持在安全的区域。</span></p> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="text-align: center;justify-content: center;margin: 10px 0%;display: flex;flex-flow: row;box-sizing: border-box;"> <section style="display: inline-block;width: auto;vertical-align: top;min-width: 10%;max-width: 100%;flex: 0 0 auto;height: auto;border-bottom: 1px solid rgb(66, 94, 255);border-bottom-right-radius: 0px;line-height: 0;align-self: flex-start;box-sizing: border-box;"> <section style="margin: 0px 0%;box-sizing: border-box;"> <section style="letter-spacing: 0px;line-height: 1.3;font-size: 21px;color: rgb(66, 94, 255);padding: 0px 3px;box-sizing: border-box;"> <p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span style=""><em style="box-sizing: border-box;"><strong style="box-sizing: border-box;"><span leaf="">END</span></strong></em></span></p> </section> </section> <section style="margin: 0px 0% -3px;box-sizing: border-box;"> <section style="display: inline-block;width: 5px;height: 5px;vertical-align: top;overflow: hidden;border-width: 0px;border-radius: 202px;border-style: none;border-color: rgb(62, 62, 62);background-color: rgb(66, 94, 255);box-sizing: border-box;"> <svg viewbox="0 0 1 1" style="float:left;line-height:0;width:0;vertical-align:top;"></svg> </section> </section> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> <section style="margin-top: 10px;margin-bottom: 10px;text-align: left;box-sizing: border-box;"> <section style="padding-left: 1em;padding-right: 1em;display: inline-block;text-align: center;box-sizing: border-box;"> <span style="display: inline-block;padding: 0.3em 0.5em;border-radius: 0.5em;background-color: rgb(65, 94, 255);color: rgb(255, 255, 255);box-sizing: border-box;" title=""><p style="margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf="">猜你喜欢</span></p></span> </section> <section style="border: 1px solid transparent;margin-top: -1em;padding: 20px 10px 10px;background-color: rgb(239, 239, 239);text-align: center;box-sizing: border-box;"> <section style="font-size: 14px;text-align: left;box-sizing: border-box;"> <ul style="list-style-type: disc;" class="list-paddingleft-1"> <li><p><span leaf=""><a class="normal_text_link" target="_blank" style="" href="https://mp.weixin.qq.com/s?__biz=MzI4NjY4MTU5Nw==&mid=2247505621&idx=1&sn=975bfe15e9309e603047a48902921be8&scene=21#wechat_redirect" textvalue="vivo HDFS EC 大规模落地实践" data-itemshowtype="0" linktype="text" data-linktype="2">vivo HDFS EC 大规模落地实践</a></span></p></li> <li><p><span leaf=""><a class="normal_text_link" target="_blank" style="" href="https://mp.weixin.qq.com/s?__biz=MzI4NjY4MTU5Nw==&mid=2247505539&idx=1&sn=65e4cf34aa3f021fc296132a74eef8af&scene=21#wechat_redirect" textvalue="vivo 浏览器福利体系架构演进之路" data-itemshowtype="0" linktype="text" data-linktype="2">vivo 浏览器福利体系架构演进之路</a></span></p></li> <li><p><span leaf=""><a class="normal_text_link" target="_blank" style="" href="https://mp.weixin.qq.com/s?__biz=MzI4NjY4MTU5Nw==&mid=2247505420&idx=1&sn=132c60505e1d58cb5b1a48a55b700054&scene=21#wechat_redirect" textvalue="慢SQL优化实战:从一例线上慢SQL探究执行引擎工作过程" data-itemshowtype="0" linktype="text" data-linktype="2">慢SQL优化实战:从一例线上慢SQL探究执行引擎工作过程</a></span></p></li> </ul> </section> </section> </section> <p style="white-space: normal;margin: 0px;padding: 0px;box-sizing: border-box;"><span leaf=""><br></span></p> </section> <section class="mp_profile_iframe_wrp" nodeleaf=""> <mp-common-profile class="js_uneditable custom_select_card mp_profile_iframe" data-pluginname="mpprofile" data-nickname="vivo互联网技术" data-alias="vivoVMIC" data-from="0" data-headimg="http://mmbiz.qpic.cn/mmbiz_png/4g5IMGibSxt45QXJZicZ9gaNU2mRSlvqhQd94MJ7oQh4QFj1ibPV66xnUiaKoicSatwaGXepL5sBDSDLEckicX1ttibHg/0?wx_fmt=png" data-signature="分享 vivo 互联网技术干货与沙龙活动,推荐最新行业动态与热门会议。" data-id="MzI4NjY4MTU5Nw==" data-is_biz_ban="0" data-service_type="2" data-verify_status="2"></mp-common-profile> </section> <p style="display: none;"> <mp-style-type data-value="3"></mp-style-type></p>