{"expand":"renderedFields,names,schema,operations,editmeta,changelog,versionedRepresentations","id":"45419","self":"https://jira.geedge.net/rest/api/2/issue/45419","key":"OMPUB-1402","fields":{"issuetype":{"self":"https://jira.geedge.net/rest/api/2/issuetype/10004","id":"10004","description":"","iconUrl":"https://jira.geedge.net/secure/viewavatar?size=xsmall&avatarId=10303&avatarType=issuetype","name":"故障","subtask":false,"avatarId":10303},"components":[],"timespent":null,"timeoriginalestimate":null,"description":"{*}现象描述：{*}M现场3个站点出现磁盘空间利用率超出80%告警，经查看每个节点存储分布不均匀：\r\n * YGN-MYTEL 的使用率在 66% 到 90% 之间\r\n * YGN-ORD 的使用率在 55% 到 82% 之间\r\n * YGN-CPN-TLN 为 的使用率在 61% 到 89% 之间\r\n\r\n出现告警时，Namenode节点的存储空间使用率比其它节点多25%左右。例如：YGN-MYTEL的Namenode节点存储16TB，其它节点存储只有12TB。\r\n\r\n ","project":{"self":"https://jira.geedge.net/rest/api/2/project/10206","id":"10206","key":"OMPUB","name":"Operation and Maintenance","projectTypeKey":"business","avatarUrls":{"48x48":"https://jira.geedge.net/secure/projectavatar?pid=10206&avatarId=10715","24x24":"https://jira.geedge.net/secure/projectavatar?size=small&pid=10206&avatarId=10715","16x16":"https://jira.geedge.net/secure/projectavatar?size=xsmall&pid=10206&avatarId=10715","32x32":"https://jira.geedge.net/secure/projectavatar?size=medium&pid=10206&avatarId=10715"},"projectCategory":{"self":"https://jira.geedge.net/rest/api/2/projectCategory/10002","id":"10002","description":"系统运维","name":"MaintenanceDev"}},"fixVersions":[],"aggregatetimespent":null,"resolution":{"self":"https://jira.geedge.net/rest/api/2/resolution/10000","id":"10000","description":"该问题的工作流程已完成。","name":"完成"},"timetracking":{},"customfield_10401":null,"customfield_10104":null,"customfield_10402":null,"customfield_10105":"0|i05uss:","customfield_10403":null,"customfield_10404":null,"attachment":[],"aggregatetimeestimate":null,"resolutiondate":"2024-08-16T16:33:58.965+0800","workratio":-1,"summary":"【M22】OLAP 三个分中心的 HDFS 节点磁盘空间利用率超过 80%","lastViewed":null,"watches":{"self":"https://jira.geedge.net/rest/api/2/issue/OMPUB-1402/watchers","watchCount":1,"isWatching":false},"creator":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"subtasks":[],"created":"2024-08-05T19:37:17.162+0800","reporter":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"customfield_10000":"{summaryBean=com.atlassian.jira.plugin.devstatus.rest.SummaryBean@5a4997a[summary={pullrequest=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@70dc4633[overall=PullRequestOverallBean{stateCount=0, state='OPEN', details=PullRequestOverallDetails{openCount=0, mergedCount=0, declinedCount=0}},byInstanceType={}], build=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@447c945[overall=com.atlassian.jira.plugin.devstatus.summary.beans.BuildOverallBean@16b8ea20[failedBuildCount=0,successfulBuildCount=0,unknownBuildCount=0,count=0,lastUpdated=<null>,lastUpdatedTimestamp=<null>],byInstanceType={}], review=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@5ec34ae[overall=com.atlassian.jira.plugin.devstatus.summary.beans.ReviewsOverallBean@6f512007[stateCount=0,state=<null>,dueDate=<null>,overDue=false,count=0,lastUpdated=<null>,lastUpdatedTimestamp=<null>],byInstanceType={}], deployment-environment=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@55e6d149[overall=com.atlassian.jira.plugin.devstatus.summary.beans.DeploymentOverallBean@45ed83f[topEnvironments=[],showProjects=false,successfulCount=0,count=0,lastUpdated=<null>,lastUpdatedTimestamp=<null>],byInstanceType={}], repository=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@60f3d55e[overall=com.atlassian.jira.plugin.devstatus.summary.beans.CommitOverallBean@6c4805f[count=0,lastUpdated=<null>,lastUpdatedTimestamp=<null>],byInstanceType={}], branch=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@65cb6557[overall=com.atlassian.jira.plugin.devstatus.summary.beans.BranchOverallBean@17a6c6f1[count=0,lastUpdated=<null>,lastUpdatedTimestamp=<null>],byInstanceType={}]},errors=[],configErrors=[]], devSummaryJson={\"cachedValue\":{\"errors\":[],\"configErrors\":[],\"summary\":{\"pullrequest\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":\"OPEN\",\"details\":{\"openCount\":0,\"mergedCount\":0,\"declinedCount\":0,\"total\":0},\"open\":true},\"byInstanceType\":{}},\"build\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"failedBuildCount\":0,\"successfulBuildCount\":0,\"unknownBuildCount\":0},\"byInstanceType\":{}},\"review\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":null,\"dueDate\":null,\"overDue\":false,\"completed\":false},\"byInstanceType\":{}},\"deployment-environment\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"topEnvironments\":[],\"showProjects\":false,\"successfulCount\":0},\"byInstanceType\":{}},\"repository\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}},\"branch\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}}}},\"isStale\":false}}","aggregateprogress":{"progress":0,"total":0},"customfield_10100":null,"priority":{"self":"https://jira.geedge.net/rest/api/2/priority/3","iconUrl":"https://jira.geedge.net/images/icons/priorities/medium.svg","name":"Medium","id":"3"},"customfield_10200":null,"customfield_10400":null,"labels":["M22"],"environment":null,"timeestimate":null,"aggregatetimeoriginalestimate":null,"versions":[],"duedate":"2024-08-16","progress":{"progress":0,"total":0},"issuelinks":[],"comment":{"comments":[{"self":"https://jira.geedge.net/rest/api/2/issue/45419/comment/84539","id":"84539","author":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"body":"问题原因：hbase的mob文件的合并导致hdfs数据不均衡。hbase的mob文件的合并由hbase的master进程负责，master将mob文件合并完成后写入hdfs，master作为hdfs的客户端，数据会优先写入本机的datanode，因此导致master进程所在服务器的datanode磁盘占用比其他节点高。\r\n\r\n ","updateAuthor":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"created":"2024-08-16T16:33:32.877+0800","updated":"2024-08-16T16:33:32.877+0800"},{"self":"https://jira.geedge.net/rest/api/2/issue/45419/comment/84540","id":"84540","author":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"body":"添加定时任务，每天凌晨01:00执行。在hdfs集群的最后一台服务器的/etc/cron.d/目录下创建run_hdfs_balancer文件，添加以下内容，修改hdfs安装目录：\r\n|#Balance hdfs data.\r\n0 1 * * * root /{color:#ff0000}{hadoop_dir}{color}/hadoop-2.7.1/sbin/start-balancer.sh -threshold 5 |","updateAuthor":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"created":"2024-08-16T16:33:58.969+0800","updated":"2024-08-16T16:33:58.969+0800"}],"maxResults":2,"total":2,"startAt":0},"votes":{"self":"https://jira.geedge.net/rest/api/2/issue/OMPUB-1402/votes","votes":0,"hasVoted":false},"worklog":{"startAt":0,"maxResults":20,"total":0,"worklogs":[]},"assignee":{"self":"https://jira.geedge.net/rest/api/2/user?username=wangchengcheng","name":"wangchengcheng","key":"JIRAUSER10603","emailAddress":"wangchengcheng@geedgenetworks.com","avatarUrls":{"48x48":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=48","24x24":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=24","16x16":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=16","32x32":"https://www.gravatar.com/avatar/9312bd750f38605f6a8d6425d20c8ad5?d=mm&s=32"},"displayName":"王成成","active":true,"timeZone":"Asia/Shanghai"},"updated":"2024-08-29T15:27:30.199+0800","status":{"self":"https://jira.geedge.net/rest/api/2/status/10103","description":"这一问题被认为是完成, 这项决议是正确的。问题已关闭可以重新开放。","iconUrl":"https://jira.geedge.net/images/icons/statuses/generic.png","name":"已关闭","id":"10103","statusCategory":{"self":"https://jira.geedge.net/rest/api/2/statuscategory/3","id":3,"key":"done","colorName":"green","name":"完成"}}}}