2025-03-16T23:55:58.985540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169358985,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2852167680,\"uptime\":1989254.37,\"totalMem\":4018151424,\"loadavg\":[0.38,0.24,0.09],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23923990,\"nice\":16920,\"sys\":3416320,\"idle\":1954538610,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23648180,\"nice\":16360,\"sys\":3432160,\"idle\":1954756390,\"irq\":0}}],\"eventLoopStats\":{\"min\":56,\"max\":30018,\"num\":3,\"sum\":60026}}}"}
2025-03-16T23:55:58.985687+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169358985,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-16T23:56:59.017536+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169419017,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2882838528,\"uptime\":1989314.4,\"totalMem\":4018151424,\"loadavg\":[0.29,0.23,0.1],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23931190,\"nice\":16920,\"sys\":3416980,\"idle\":1954590610,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23652170,\"nice\":16360,\"sys\":3432470,\"idle\":1954811820,\"irq\":0}}],\"eventLoopStats\":{\"min\":47,\"max\":30001,\"num\":3,\"sum\":60029}}}"}
2025-03-16T23:56:59.017855+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169419017,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-16T23:58:59.053475+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169539053,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2915983360,\"uptime\":1989434.44,\"totalMem\":4018151424,\"loadavg\":[0.23,0.26,0.13],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23951550,\"nice\":16920,\"sys\":3417880,\"idle\":1954688870,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23671160,\"nice\":16360,\"sys\":3433330,\"idle\":1954911350,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29960,\"num\":13,\"sum\":60006}}}"}
2025-03-16T23:58:59.053595+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169539053,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-16T23:59:59.081466+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169599081,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2915397632,\"uptime\":1989494.47,\"totalMem\":4018151424,\"loadavg\":[0.12,0.23,0.12],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23952340,\"nice\":16920,\"sys\":3417930,\"idle\":1954748130,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23671770,\"nice\":16360,\"sys\":3433390,\"idle\":1954970670,\"irq\":0}}],\"eventLoopStats\":{\"min\":39,\"max\":30002,\"num\":3,\"sum\":60026}}}"}
2025-03-16T23:59:59.081586+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169599081,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:00:59.113453+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169659113,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2908020736,\"uptime\":1989554.5,\"totalMem\":4018151424,\"loadavg\":[0.04,0.18,0.11],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23953230,\"nice\":16950,\"sys\":3418020,\"idle\":1954807070,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23672330,\"nice\":16410,\"sys\":3433560,\"idle\":1955029700,\"irq\":0}}],\"eventLoopStats\":{\"min\":18,\"max\":30027,\"num\":3,\"sum\":60030}}}"}
2025-03-17T00:00:59.113600+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169659113,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:04:59.217486+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169899217,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2912546816,\"uptime\":1989794.6,\"totalMem\":4018151424,\"loadavg\":[0,0.07,0.08],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23954730,\"nice\":16950,\"sys\":3418310,\"idle\":1955044130,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23673940,\"nice\":16410,\"sys\":3433900,\"idle\":1955266790,\"irq\":0}}],\"eventLoopStats\":{\"min\":32,\"max\":30027,\"num\":3,\"sum\":60029}}}"}
2025-03-17T00:04:59.217698+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169899217,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:05:59.249459+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742169959249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2909741056,\"uptime\":1989854.63,\"totalMem\":4018151424,\"loadavg\":[0,0.06,0.07],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23955240,\"nice\":16950,\"sys\":3418430,\"idle\":1955103430,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23674550,\"nice\":16410,\"sys\":3433980,\"idle\":1955326000,\"irq\":0}}],\"eventLoopStats\":{\"min\":30,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T00:05:59.249578+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742169959249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:08:59.277490+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170139277,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2909650944,\"uptime\":1990034.66,\"totalMem\":4018151424,\"loadavg\":[0,0.03,0.05],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23956690,\"nice\":16950,\"sys\":3418740,\"idle\":1955281060,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23675720,\"nice\":16410,\"sys\":3434230,\"idle\":1955504010,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29972,\"num\":13,\"sum\":59995}}}"}
2025-03-17T00:08:59.277568+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170139277,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:16:59.457446+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170619457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2911035392,\"uptime\":1990514.84,\"totalMem\":4018151424,\"loadavg\":[0,0,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23961100,\"nice\":16950,\"sys\":3419400,\"idle\":1955754590,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23679310,\"nice\":16410,\"sys\":3434910,\"idle\":1955977830,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30027,\"num\":3,\"sum\":60029}}}"}
2025-03-17T00:16:59.457517+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170619457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:18:03.486152+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170683485,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T00:18:03.487027+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170683486,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T00:18:59.509462+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170739509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2910908416,\"uptime\":1990634.89,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23962010,\"nice\":16950,\"sys\":3419570,\"idle\":1955873090,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23680050,\"nice\":16410,\"sys\":3435130,\"idle\":1956096390,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29972,\"num\":13,\"sum\":60024}}}"}
2025-03-17T00:18:59.509524+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170739509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:20:59.529486+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170859529,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2909782016,\"uptime\":1990754.91,\"totalMem\":4018151424,\"loadavg\":[0.2,0.09,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23963330,\"nice\":16950,\"sys\":3419740,\"idle\":1955991550,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23681430,\"nice\":16410,\"sys\":3435350,\"idle\":1956214840,\"irq\":0}}],\"eventLoopStats\":{\"min\":17,\"max\":30001,\"num\":3,\"sum\":60001}}}"}
2025-03-17T00:20:59.529554+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170859529,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:21:59.557459+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170919557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2908639232,\"uptime\":1990814.94,\"totalMem\":4018151424,\"loadavg\":[0.11,0.09,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23963930,\"nice\":16950,\"sys\":3419790,\"idle\":1956050820,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23682180,\"nice\":16410,\"sys\":3435440,\"idle\":1956274090,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30030,\"num\":3,\"sum\":60026}}}"}
2025-03-17T00:21:59.557526+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170919557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:22:59.577439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742170979577,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2908098560,\"uptime\":1990874.96,\"totalMem\":4018151424,\"loadavg\":[0.04,0.07,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23964560,\"nice\":16950,\"sys\":3419880,\"idle\":1956110070,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23682670,\"nice\":16410,\"sys\":3435520,\"idle\":1956333380,\"irq\":0}}],\"eventLoopStats\":{\"min\":22,\"max\":30013,\"num\":3,\"sum\":60018}}}"}
2025-03-17T00:22:59.577525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742170979577,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:29:59.753681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742171399753,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890252288,\"uptime\":1991295.14,\"totalMem\":4018151424,\"loadavg\":[0.07,0.32,0.22],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24000390,\"nice\":16950,\"sys\":3440350,\"idle\":1956472270,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23718600,\"nice\":16410,\"sys\":3456240,\"idle\":1956695270,\"irq\":0}}],\"eventLoopStats\":{\"min\":36,\"max\":30030,\"num\":3,\"sum\":60028}}}"}
2025-03-17T00:29:59.753685+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742171399753,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:30:59.773468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742171459773,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897027072,\"uptime\":1991355.16,\"totalMem\":4018151424,\"loadavg\":[0.02,0.26,0.2],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24000950,\"nice\":16950,\"sys\":3440470,\"idle\":1956531540,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23719130,\"nice\":16410,\"sys\":3456320,\"idle\":1956754470,\"irq\":0}}],\"eventLoopStats\":{\"min\":37,\"max\":30011,\"num\":3,\"sum\":60017}}}"}
2025-03-17T00:30:59.773538+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742171459773,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:39:59.941411+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742171999941,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2895949824,\"uptime\":1991895.33,\"totalMem\":4018151424,\"loadavg\":[0,0.05,0.1],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24005870,\"nice\":16950,\"sys\":3441130,\"idle\":1957064990,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23724100,\"nice\":16410,\"sys\":3457040,\"idle\":1957287830,\"irq\":0}}],\"eventLoopStats\":{\"min\":75,\"max\":30028,\"num\":3,\"sum\":60028}}}"}
2025-03-17T00:39:59.941459+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742171999941,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:42:00.001481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742172120001,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894639104,\"uptime\":1992015.39,\"totalMem\":4018151424,\"loadavg\":[0.08,0.07,0.1],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24006780,\"nice\":16950,\"sys\":3441260,\"idle\":1957183510,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23724970,\"nice\":16410,\"sys\":3457200,\"idle\":1957406350,\"irq\":0}}],\"eventLoopStats\":{\"min\":47,\"max\":30010,\"num\":3,\"sum\":60030}}}"}
2025-03-17T00:42:00.001591+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742172120001,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:43:00.021443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742172180021,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894422016,\"uptime\":1992075.4,\"totalMem\":4018151424,\"loadavg\":[0.03,0.05,0.09],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24007220,\"nice\":16950,\"sys\":3441330,\"idle\":1957242760,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23725260,\"nice\":16410,\"sys\":3457250,\"idle\":1957465660,\"irq\":0}}],\"eventLoopStats\":{\"min\":27,\"max\":30001,\"num\":3,\"sum\":60017}}}"}
2025-03-17T00:43:00.021503+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742172180021,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:45:00.053413+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742172300053,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2895544320,\"uptime\":1992195.43,\"totalMem\":4018151424,\"loadavg\":[0.06,0.05,0.08],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24008340,\"nice\":16950,\"sys\":3441520,\"idle\":1957360950,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23726320,\"nice\":16410,\"sys\":3457410,\"idle\":1957583820,\"irq\":0}}],\"eventLoopStats\":{\"min\":29974,\"max\":30027,\"num\":2,\"sum\":60001}}}"}
2025-03-17T00:45:00.053476+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742172300053,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:46:00.081491+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742172360081,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894565376,\"uptime\":1992255.46,\"totalMem\":4018151424,\"loadavg\":[0.02,0.04,0.08],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24008710,\"nice\":16950,\"sys\":3441630,\"idle\":1957420240,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23726670,\"nice\":16410,\"sys\":3457480,\"idle\":1957642990,\"irq\":0}}],\"eventLoopStats\":{\"min\":25,\"max\":30029,\"num\":3,\"sum\":60026}}}"}
2025-03-17T00:46:00.081642+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742172360081,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:57:00.305440+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173020305,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888605696,\"uptime\":1992915.69,\"totalMem\":4018151424,\"loadavg\":[0.07,0.02,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24014240,\"nice\":16950,\"sys\":3442410,\"idle\":1958071550,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23731500,\"nice\":16410,\"sys\":3458420,\"idle\":1958295030,\"irq\":0}}],\"eventLoopStats\":{\"min\":23,\"max\":30010,\"num\":3,\"sum\":60030}}}"}
2025-03-17T00:57:00.305572+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173020305,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T00:58:11.500714+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173091500,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T00:58:11.501599+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173091501,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T01:00:00.385452+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173200385,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889928704,\"uptime\":1993095.77,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24015420,\"nice\":16950,\"sys\":3442620,\"idle\":1958249210,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23732470,\"nice\":16410,\"sys\":3458630,\"idle\":1958473080,\"irq\":0}}],\"eventLoopStats\":{\"min\":29996,\"max\":30032,\"num\":2,\"sum\":60028}}}"}
2025-03-17T01:00:00.385509+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173200385,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:01:00.409375+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173260409,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888663040,\"uptime\":1993155.79,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24015920,\"nice\":16950,\"sys\":3442730,\"idle\":1958308370,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23732880,\"nice\":16410,\"sys\":3458760,\"idle\":1958532410,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30030,\"num\":3,\"sum\":60025}}}"}
2025-03-17T01:01:00.409513+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173260409,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:04:00.469427+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173440469,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889383936,\"uptime\":1993335.85,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24018290,\"nice\":16950,\"sys\":3442960,\"idle\":1958486000,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23734340,\"nice\":16410,\"sys\":3458970,\"idle\":1958710380,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29982,\"num\":12,\"sum\":60002}}}"}
2025-03-17T01:04:00.469471+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173440469,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:07:00.521424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173620521,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894114816,\"uptime\":1993515.9,\"totalMem\":4018151424,\"loadavg\":[0.06,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24020340,\"nice\":16950,\"sys\":3443180,\"idle\":1958663820,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23736430,\"nice\":16410,\"sys\":3459200,\"idle\":1958888180,\"irq\":0}}],\"eventLoopStats\":{\"min\":28,\"max\":30026,\"num\":3,\"sum\":60026}}}"}
2025-03-17T01:07:00.521545+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173620521,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:10:00.589421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742173800589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893406208,\"uptime\":1993695.97,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24021970,\"nice\":16950,\"sys\":3443410,\"idle\":1958841640,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23738360,\"nice\":16410,\"sys\":3459370,\"idle\":1959065920,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30005,\"num\":3,\"sum\":60026}}}"}
2025-03-17T01:10:00.589477+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742173800589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:14:00.653525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742174040653,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892677120,\"uptime\":1993936.04,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24023400,\"nice\":16950,\"sys\":3443730,\"idle\":1959078810,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23740590,\"nice\":16410,\"sys\":3459720,\"idle\":1959302690,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29982,\"num\":14,\"sum\":60003}}}"}
2025-03-17T01:14:00.653540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742174040653,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:18:15.224209+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742174295224,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T01:18:15.225033+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742174295224,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T01:23:16.309844+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742174596309,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T01:23:16.310641+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742174596310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T01:40:52.540955+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742175652540,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T01:40:52.541775+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742175652541,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T01:41:01.157463+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742175661157,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2896535552,\"uptime\":1995556.54,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24034590,\"nice\":16950,\"sys\":3445970,\"idle\":1960677800,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23752430,\"nice\":16410,\"sys\":3461920,\"idle\":1960901900,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30029,\"num\":13,\"sum\":60000}}}"}
2025-03-17T01:41:01.157596+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742175661157,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:43:01.189472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742175781189,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2896953344,\"uptime\":1995676.57,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24035390,\"nice\":16950,\"sys\":3446110,\"idle\":1960796330,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23753240,\"nice\":16410,\"sys\":3462090,\"idle\":1961020440,\"irq\":0}}],\"eventLoopStats\":{\"min\":70,\"max\":30011,\"num\":3,\"sum\":60004}}}"}
2025-03-17T01:43:01.189532+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742175781189,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:47:01.281437+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742176021281,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897051648,\"uptime\":1995916.66,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24037760,\"nice\":16950,\"sys\":3446480,\"idle\":1961032850,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23755120,\"nice\":16410,\"sys\":3462440,\"idle\":1961257210,\"irq\":0}}],\"eventLoopStats\":{\"min\":60,\"max\":30023,\"num\":3,\"sum\":60030}}}"}
2025-03-17T01:47:01.281505+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742176021281,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:48:01.309441+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742176081309,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2896838656,\"uptime\":1995976.69,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24038530,\"nice\":16950,\"sys\":3446570,\"idle\":1961092120,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23755870,\"nice\":16410,\"sys\":3462530,\"idle\":1961316500,\"irq\":0}}],\"eventLoopStats\":{\"min\":52,\"max\":30014,\"num\":3,\"sum\":60028}}}"}
2025-03-17T01:48:01.309582+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742176081309,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:50:01.349411+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742176201349,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897477632,\"uptime\":1996096.73,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24040380,\"nice\":16950,\"sys\":3446720,\"idle\":1961210570,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23756880,\"nice\":16410,\"sys\":3462660,\"idle\":1961435150,\"irq\":0}}],\"eventLoopStats\":{\"min\":26,\"max\":30029,\"num\":3,\"sum\":60017}}}"}
2025-03-17T01:50:01.349540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742176201349,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:56:01.413428+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742176561413,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2895179776,\"uptime\":1996456.8,\"totalMem\":4018151424,\"loadavg\":[0.03,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24043630,\"nice\":16950,\"sys\":3447210,\"idle\":1961565890,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23759800,\"nice\":16410,\"sys\":3463190,\"idle\":1961790650,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30027,\"num\":13,\"sum\":60001}}}"}
2025-03-17T01:56:01.413496+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742176561413,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T01:58:01.461422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742176681461,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894974976,\"uptime\":1996576.84,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24044550,\"nice\":16950,\"sys\":3447370,\"idle\":1961684450,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23760710,\"nice\":16410,\"sys\":3463380,\"idle\":1961909180,\"irq\":0}}],\"eventLoopStats\":{\"min\":92,\"max\":30030,\"num\":3,\"sum\":60020}}}"}
2025-03-17T01:58:01.461575+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742176681461,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:08:01.677451+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742177281677,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2899267584,\"uptime\":1997177.06,\"totalMem\":4018151424,\"loadavg\":[0.56,0.12,0.04],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24048890,\"nice\":16950,\"sys\":3448230,\"idle\":1962277040,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23765520,\"nice\":16410,\"sys\":3464220,\"idle\":1962501680,\"irq\":0}}],\"eventLoopStats\":{\"min\":46,\"max\":30006,\"num\":3,\"sum\":60026}}}"}
2025-03-17T02:08:01.677514+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742177281677,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:10:01.733429+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742177401733,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2898894848,\"uptime\":1997297.12,\"totalMem\":4018151424,\"loadavg\":[0.21,0.13,0.05],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24050010,\"nice\":16950,\"sys\":3448380,\"idle\":1962395480,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23766340,\"nice\":16410,\"sys\":3464340,\"idle\":1962620310,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30030,\"num\":3,\"sum\":60026}}}"}
2025-03-17T02:10:01.733564+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742177401733,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:13:01.766259+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742177581766,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897174528,\"uptime\":1997477.17,\"totalMem\":4018151424,\"loadavg\":[0.01,0.06,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24051410,\"nice\":16950,\"sys\":3448570,\"idle\":1962573130,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23767480,\"nice\":16410,\"sys\":3464580,\"idle\":1962798070,\"irq\":0}}],\"eventLoopStats\":{\"min\":23,\"max\":30009,\"num\":3,\"sum\":60019}}}"}
2025-03-17T02:13:01.766343+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742177581766,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:19:01.865420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742177941865,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892644352,\"uptime\":1997837.25,\"totalMem\":4018151424,\"loadavg\":[0.08,0.07,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24054300,\"nice\":16950,\"sys\":3449090,\"idle\":1962928090,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23770180,\"nice\":16410,\"sys\":3465210,\"idle\":1963153040,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29947,\"num\":24,\"sum\":60014}}}"}
2025-03-17T02:19:01.865574+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742177941865,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:20:01.897460+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178001897,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892660736,\"uptime\":1997897.28,\"totalMem\":4018151424,\"loadavg\":[0.03,0.06,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24054680,\"nice\":16950,\"sys\":3449170,\"idle\":1962987300,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23770440,\"nice\":16410,\"sys\":3465290,\"idle\":1963212350,\"irq\":0}}],\"eventLoopStats\":{\"min\":52,\"max\":30000,\"num\":3,\"sum\":60027}}}"}
2025-03-17T02:20:01.897521+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178001897,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:27:02.057455+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178422057,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2898972672,\"uptime\":1998317.44,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24057810,\"nice\":16950,\"sys\":3449650,\"idle\":1963402360,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23774040,\"nice\":16410,\"sys\":3465810,\"idle\":1963626680,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30025,\"num\":3,\"sum\":60006}}}"}
2025-03-17T02:27:02.057597+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178422057,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:28:11.571652+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178491571,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T02:28:11.572401+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178491572,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T02:29:02.089415+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178542089,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897756160,\"uptime\":1998437.47,\"totalMem\":4018151424,\"loadavg\":[0.03,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24058740,\"nice\":16950,\"sys\":3449870,\"idle\":1963520880,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23775480,\"nice\":16410,\"sys\":3465960,\"idle\":1963745140,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29983,\"num\":13,\"sum\":59999}}}"}
2025-03-17T02:29:02.089475+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178542089,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:30:02.097430+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178602097,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897760256,\"uptime\":1998497.48,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24059200,\"nice\":16950,\"sys\":3449960,\"idle\":1963580110,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23776180,\"nice\":16410,\"sys\":3466020,\"idle\":1963804400,\"irq\":0}}],\"eventLoopStats\":{\"min\":18,\"max\":30030,\"num\":3,\"sum\":60007}}}"}
2025-03-17T02:30:02.097589+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178602097,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:31:02.125473+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178662125,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894528512,\"uptime\":1998557.51,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24059550,\"nice\":16950,\"sys\":3450060,\"idle\":1963639350,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23776740,\"nice\":16410,\"sys\":3466140,\"idle\":1963863570,\"irq\":0}}],\"eventLoopStats\":{\"min\":40,\"max\":30023,\"num\":3,\"sum\":60025}}}"}
2025-03-17T02:31:02.125528+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178662125,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:33:02.181696+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178782181,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892443648,\"uptime\":1998677.56,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24060380,\"nice\":16950,\"sys\":3450280,\"idle\":1963757830,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23777890,\"nice\":16410,\"sys\":3466320,\"idle\":1963982070,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30027,\"num\":3,\"sum\":60027}}}"}
2025-03-17T02:33:02.181704+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178782181,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:34:02.209439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742178842209,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892447744,\"uptime\":1998737.59,\"totalMem\":4018151424,\"loadavg\":[0.04,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24060920,\"nice\":16950,\"sys\":3450360,\"idle\":1963817080,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23778480,\"nice\":16410,\"sys\":3466420,\"idle\":1964041290,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29988,\"num\":13,\"sum\":60023}}}"}
2025-03-17T02:34:02.209509+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742178842209,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:37:02.265494+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179022265,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892496896,\"uptime\":1998917.65,\"totalMem\":4018151424,\"loadavg\":[0.01,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24062130,\"nice\":16950,\"sys\":3450660,\"idle\":1963994930,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23780280,\"nice\":16410,\"sys\":3466640,\"idle\":1964218930,\"irq\":0}}],\"eventLoopStats\":{\"min\":29982,\"max\":30028,\"num\":2,\"sum\":60010}}}"}
2025-03-17T02:37:02.265562+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179022265,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:38:13.394516+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179093394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T02:38:13.395383+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179093395,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T02:41:02.349412+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179262349,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892820480,\"uptime\":1999157.73,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24063840,\"nice\":16950,\"sys\":3450920,\"idle\":1964231930,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23781610,\"nice\":16410,\"sys\":3466960,\"idle\":1964456070,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30027,\"num\":3,\"sum\":60014}}}"}
2025-03-17T02:41:02.349481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179262349,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:46:40.037017+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179600036,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T02:46:40.037236+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-page-request' for consumer group='icc-reporting'"}
2025-03-17T02:46:40.037309+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T02:46:40.027Z\",\"user\":{\"id\":\"04bc90bb-223a-4af7-8f24-095137150138\",\"email\":\"jakkaldevi@salesforce.com\",\"active\":true,\"name\":\"Jagruthi Akkaldevi\",\"firstName\":\"Jagruthi\",\"lastName\":\"Akkaldevi\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE0000040HMjYAM\",\"slackUserId\":\"U0329QR7H0W\",\"createdBy\":\"jakkaldevi@salesforce.com\",\"modifiedBy\":\"jakkaldevi@salesforce.com\",\"createdAt\":\"2022-06-29T02:42:05.592Z\",\"modifiedAt\":\"2022-06-29T02:42:05.592Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"ok\":true,\"id\":\"6762fa7d-089e-4521-aace-cc25be2fcb4f\",\"serviceId\":\"PEO7ZIC\",\"serviceName\":\"Edge Application Delivery\",\"customMessage\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: Customers seeing latency on OMNI chat - single pod\\nSeverity: Sev2\\nSlack Channel: #icc-73095026\\nBridge URL: https://meet.google.com/hbh-zkju-hsu\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\\nAdditional Details: \",\"incidentSubject\":\"Customers seeing latency on OMNI chat - single pod\",\"incidentDescription\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\"},\"requestId\":\"qgm2zk-4hrvjk-8y6asc-r06mbb-1742179599589\",\"client\":\"icc-ui\"}"}
2025-03-17T02:46:40.037313+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T02:46:40.037398+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T02:46:40.037488+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:46:40.037489+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-page-request"}
2025-03-17T02:46:40.037518+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-page-request is not consumed by icc-reporting."}
2025-03-17T02:46:40.037562+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179600037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-page-request correctly routed"}
2025-03-17T02:46:41.089089+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179601088,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-page-response correctly routed"}
2025-03-17T02:52:02.573436+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179922573,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892640256,\"uptime\":1999817.96,\"totalMem\":4018151424,\"loadavg\":[0.03,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24069730,\"nice\":16950,\"sys\":3451910,\"idle\":1964883350,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23786920,\"nice\":16410,\"sys\":3467930,\"idle\":1965107390,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30004,\"num\":15,\"sum\":60009}}}"}
2025-03-17T02:52:02.573481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179922573,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:53:02.593428+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742179982593,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892783616,\"uptime\":1999877.98,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24070080,\"nice\":16950,\"sys\":3451980,\"idle\":1964942550,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23787320,\"nice\":16410,\"sys\":3467990,\"idle\":1965166670,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30023,\"num\":3,\"sum\":60020}}}"}
2025-03-17T02:53:02.593559+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742179982593,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:54:02.621425+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742180042621,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892427264,\"uptime\":1999938,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24070490,\"nice\":16950,\"sys\":3452040,\"idle\":1965001800,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23787720,\"nice\":16410,\"sys\":3468070,\"idle\":1965225970,\"irq\":0}}],\"eventLoopStats\":{\"min\":23,\"max\":30027,\"num\":3,\"sum\":60026}}}"}
2025-03-17T02:54:02.621488+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180042621,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T02:56:42.479235+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180202479,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T02:56:42.480077+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180202479,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T03:04:02.793387+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742180642793,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891210752,\"uptime\":2000538.18,\"totalMem\":4018151424,\"loadavg\":[0.07,0.03,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24074590,\"nice\":16950,\"sys\":3452900,\"idle\":1965594300,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23792630,\"nice\":16410,\"sys\":3468870,\"idle\":1965818360,\"irq\":0}}],\"eventLoopStats\":{\"min\":36,\"max\":30029,\"num\":3,\"sum\":60007}}}"}
2025-03-17T03:04:02.793516+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180642793,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:05:02.821427+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742180702821,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890604544,\"uptime\":2000598.2,\"totalMem\":4018151424,\"loadavg\":[0.02,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24075130,\"nice\":16950,\"sys\":3452980,\"idle\":1965653570,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23793130,\"nice\":16410,\"sys\":3468980,\"idle\":1965877500,\"irq\":0}}],\"eventLoopStats\":{\"min\":58,\"max\":30007,\"num\":3,\"sum\":60025}}}"}
2025-03-17T03:05:02.821491+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180702821,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:08:02.857392+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742180882857,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890543104,\"uptime\":2000778.24,\"totalMem\":4018151424,\"loadavg\":[0.05,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24076820,\"nice\":16950,\"sys\":3453310,\"idle\":1965831370,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23795120,\"nice\":16410,\"sys\":3469260,\"idle\":1966055120,\"irq\":0}}],\"eventLoopStats\":{\"min\":44,\"max\":30030,\"num\":3,\"sum\":60025}}}"}
2025-03-17T03:08:02.857523+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742180882857,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:11:02.909468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742181062909,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893852672,\"uptime\":2000958.29,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24078320,\"nice\":16950,\"sys\":3453620,\"idle\":1966009150,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23797160,\"nice\":16410,\"sys\":3469570,\"idle\":1966232770,\"irq\":0}}],\"eventLoopStats\":{\"min\":39,\"max\":30000,\"num\":3,\"sum\":60012}}}"}
2025-03-17T03:11:02.909515+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742181062909,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:16:02.957409+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742181362957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886389760,\"uptime\":2001258.34,\"totalMem\":4018151424,\"loadavg\":[0,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24080820,\"nice\":16950,\"sys\":3454050,\"idle\":1966304930,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23799650,\"nice\":16410,\"sys\":3470040,\"idle\":1966528710,\"irq\":0}}],\"eventLoopStats\":{\"min\":41,\"max\":30027,\"num\":3,\"sum\":60000}}}"}
2025-03-17T03:16:02.957560+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742181362957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:19:03.013423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742181543013,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885697536,\"uptime\":2001438.4,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24082420,\"nice\":16950,\"sys\":3454280,\"idle\":1966482630,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23800970,\"nice\":16410,\"sys\":3470230,\"idle\":1966706510,\"irq\":0}}],\"eventLoopStats\":{\"min\":67,\"max\":30029,\"num\":3,\"sum\":60024}}}"}
2025-03-17T03:19:03.013548+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742181543013,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:26:06.350544+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742181966350,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T03:26:06.351342+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742181966351,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T03:30:03.221422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742182203221,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885591040,\"uptime\":2002098.6,\"totalMem\":4018151424,\"loadavg\":[0.03,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24089110,\"nice\":16950,\"sys\":3455230,\"idle\":1967134460,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23807830,\"nice\":16410,\"sys\":3471130,\"idle\":1967358070,\"irq\":0}}],\"eventLoopStats\":{\"min\":83,\"max\":30000,\"num\":3,\"sum\":60019}}}"}
2025-03-17T03:30:03.221503+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742182203221,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:31:03.249403+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742182263249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885599232,\"uptime\":2002158.63,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24089470,\"nice\":16950,\"sys\":3455350,\"idle\":1967193690,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23808200,\"nice\":16410,\"sys\":3471230,\"idle\":1967417290,\"irq\":0}}],\"eventLoopStats\":{\"min\":63,\"max\":30023,\"num\":3,\"sum\":60028}}}"}
2025-03-17T03:31:03.249483+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742182263249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:41:09.483613+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742182869483,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T03:41:09.484449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742182869484,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T03:42:03.481422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742182923481,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884915200,\"uptime\":2002818.87,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24093970,\"nice\":16950,\"sys\":3456300,\"idle\":1967845530,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23813010,\"nice\":16410,\"sys\":3472130,\"idle\":1968069000,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29929,\"num\":13,\"sum\":60026}}}"}
2025-03-17T03:42:03.481539+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742182923481,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:45:03.557423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742183103557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884198400,\"uptime\":2002998.94,\"totalMem\":4018151424,\"loadavg\":[0.02,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24095470,\"nice\":16950,\"sys\":3456500,\"idle\":1968023380,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23814940,\"nice\":16410,\"sys\":3472330,\"idle\":1968246650,\"irq\":0}}],\"eventLoopStats\":{\"min\":32,\"max\":30000,\"num\":3,\"sum\":60015}}}"}
2025-03-17T03:45:03.557498+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183103557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:46:10.572441+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183170572,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T03:46:10.573273+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183170573,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T03:48:03.637394+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742183283637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891816960,\"uptime\":2003179.02,\"totalMem\":4018151424,\"loadavg\":[0.03,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24096810,\"nice\":16950,\"sys\":3456770,\"idle\":1968200840,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23816620,\"nice\":16410,\"sys\":3472570,\"idle\":1968424020,\"irq\":0}}],\"eventLoopStats\":{\"min\":18,\"max\":30027,\"num\":3,\"sum\":60028}}}"}
2025-03-17T03:48:03.637538+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183283637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:53:03.729424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742183583729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892259328,\"uptime\":2003479.11,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24099590,\"nice\":16950,\"sys\":3457110,\"idle\":1968497190,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23818890,\"nice\":16410,\"sys\":3472920,\"idle\":1968720160,\"irq\":0}}],\"eventLoopStats\":{\"min\":46,\"max\":30025,\"num\":3,\"sum\":60019}}}"}
2025-03-17T03:53:03.729472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183583729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:55:03.781571+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742183703781,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889273344,\"uptime\":2003599.17,\"totalMem\":4018151424,\"loadavg\":[0.03,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24100520,\"nice\":16950,\"sys\":3457260,\"idle\":1968615730,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23819870,\"nice\":16410,\"sys\":3473050,\"idle\":1968838670,\"irq\":0}}],\"eventLoopStats\":{\"min\":1,\"max\":29986,\"num\":9,\"sum\":60020}}}"}
2025-03-17T03:55:03.781624+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183703781,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:59:03.861441+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742183943861,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889605120,\"uptime\":2003839.25,\"totalMem\":4018151424,\"loadavg\":[0.01,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24102670,\"nice\":16950,\"sys\":3457560,\"idle\":1968852650,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23821950,\"nice\":16410,\"sys\":3473410,\"idle\":1969075870,\"irq\":0}}],\"eventLoopStats\":{\"min\":32,\"max\":30008,\"num\":3,\"sum\":60022}}}"}
2025-03-17T03:59:03.861512+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183943861,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T03:59:12.320219+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183952320,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T03:59:12.320943+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742183952320,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T04:00:00.126499+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8fcf7772-e019-4725-a77e-724a972a256e"}
2025-03-17T04:00:00.126579+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='bullmq-job-completed' for consumer group='icc-reporting'"}
2025-03-17T04:00:00.126630+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T04:00:00.077Z\",\"incident\":{\"id\":\"8fcf7772-e019-4725-a77e-724a972a256e\"},\"eventObj\":{\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"reminderId\":\"631782ef-7283-40a9-96fd-9ec9a0b3d406\",\"message\":\"Close the case\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"}},\"client\":\"icc-rest\"}"}
2025-03-17T04:00:00.126632+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T04:00:00.126705+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T04:00:00.126818+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:00:00.126819+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8fcf7772-e019-4725-a77e-724a972a256e - bullmq-job-completed"}
2025-03-17T04:00:00.126836+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type bullmq-job-completed is not consumed by icc-reporting."}
2025-03-17T04:00:00.126897+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8fcf7772-e019-4725-a77e-724a972a256e - bullmq-job-completed correctly routed"}
2025-03-17T04:00:00.126967+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000126,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8fcf7772-e019-4725-a77e-724a972a256e"}
2025-03-17T04:00:00.127050+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='bullmq-job-completed' for consumer group='icc-reporting'"}
2025-03-17T04:00:00.127066+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T04:00:00.076Z\",\"incident\":{\"id\":\"8fcf7772-e019-4725-a77e-724a972a256e\"},\"eventObj\":{\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"reminderId\":\"6b146ee2-81a9-449b-8d58-d72a8967f969\",\"message\":\"If the Incident is invalid, close using /icc-incident quickclose in slack or the overflow menu in the Web UI\",\"additionalInfo\":{\"rule\":{\"name\":\"Low Severity Incident Reminders\",\"type\":\"team\"},\"status\":\"InProgress\"}},\"client\":\"icc-rest\"}"}
2025-03-17T04:00:00.127098+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T04:00:00.127126+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T04:00:00.127194+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:00:00.127197+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8fcf7772-e019-4725-a77e-724a972a256e - bullmq-job-completed"}
2025-03-17T04:00:00.127233+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type bullmq-job-completed is not consumed by icc-reporting."}
2025-03-17T04:00:00.127234+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000127,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8fcf7772-e019-4725-a77e-724a972a256e - bullmq-job-completed correctly routed"}
2025-03-17T04:00:00.175930+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000175,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8fcf7772-e019-4725-a77e-724a972a256e"}
2025-03-17T04:00:00.176128+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-reminder-triggered' for consumer group='icc-reporting'"}
2025-03-17T04:00:00.176199+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T04:00:00.166Z\",\"incident\":{\"id\":\"8fcf7772-e019-4725-a77e-724a972a256e\"},\"eventObj\":{\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"reminderId\":\"631782ef-7283-40a9-96fd-9ec9a0b3d406\",\"message\":\"Close the case\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"reminderData\":{\"id\":\"631782ef-7283-40a9-96fd-9ec9a0b3d406\",\"type\":\"REMINDER\",\"active\":true,\"message\":\"Close the case\",\"audience\":[],\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"offsetUnit\":\"h\",\"recurrence\":50,\"triggersAt\":\"2025-03-17T11:47:13.042Z\",\"isRecurring\":true,\"occurrences\":[{\"oid\":\"c992b331-5a4e-48ef-9fa0-7738d049ffd6\",\"completed\":false,\"triggeredAt\":\"2025-03-15T07:47:13.211Z\"},{\"oid\":\"b818523c-42ea-4778-8233-37721407f6b6\",\"completed\":false,\"triggeredAt\":\"2025-03-15T08:00:00.125Z\"},{\"oid\":\"c1198cd7-102c-42b6-9bbe-a5107aeeddbf\",\"completed\":false,\"triggeredAt\":\"2025-03-15T12:00:00.168Z\"},{\"oid\":\"60d269c9-da7a-4dcd-8e55-0a3b116fbda1\",\"completed\":false,\"triggeredAt\":\"2025-03-15T16:00:00.128Z\"},{\"oid\":\"69b49a23-5312-4f25-9681-ade43e73c433\",\"completed\":false,\"triggeredAt\":\"2025-03-15T20:00:00.160Z\"},{\"oid\":\"8380230a-2f13-4452-92d7-c0ec882f9aa1\",\"completed\":false,\"triggeredAt\":\"2025-03-16T00:00:00.102Z\"},{\"oid\":\"3a9cb0eb-4d7d-497c-9407-3de2a9ed30c5\",\"completed\":false,\"triggeredAt\":\"2025-03-16T04:00:00.993Z\"},{\"oid\":\"3d1f57d6-1971-4e6e-9c6d-50bec5fdf38f\",\"completed\":false,\"triggeredAt\":\"2025-03-16T08:00:00.192Z\"},{\"oid\":\"f1ba6de0-7f3d-4d5e-83f7-48c0e2c97aa9\",\"completed\":false,\"triggeredAt\":\"2025-03-16T12:00:00.311Z\"},{\"oid\":\"29c72c65-c5cb-4f1e-befc-374a5eaf2670\",\"completed\":false,\"triggeredAt\":\"2025-03-16T16:00:00.128Z\"},{\"oid\":\"ffb9708e-1f9a-422f-8798-8cb52bdddb3c\",\"completed\":false,\"triggeredAt\":\"2025-03-16T20:00:00.189Z\"},{\"oid\":\"d2fe9fa0-2b0f-4d4f-a1c2-ae4b8d605447\",\"completed\":false,\"triggeredAt\":\"2025-03-17T00:00:00.146Z\"},{\"oid\":\"c0d3bf76-7a8c-41c5-a568-ed0abad4e4dd\",\"triggeredAt\":\"2025-03-17T04:00:00.145Z\",\"completed\":false}],\"offsetValue\":1,\"frequencyUnit\":\"h\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"frequencyValue\":4,\"currentOccurrence\":13,\"currentOccurrenceId\":\"c0d3bf76-7a8c-41c5-a568-ed0abad4e4dd\",\"inheritedFromTeamConfig\":true},\"slackChannelId\":\"C08J5R8VCGH\",\"nextDueAt\":\"2025-03-17T11:47:13.042Z\"}}"}
2025-03-17T04:00:00.176203+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T04:00:00.176261+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T04:00:00.176355+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:00:00.176365+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8fcf7772-e019-4725-a77e-724a972a256e - icc-reminder-triggered"}
2025-03-17T04:00:00.176366+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-reminder-triggered is not consumed by icc-reporting."}
2025-03-17T04:00:00.176418+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000176,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8fcf7772-e019-4725-a77e-724a972a256e - icc-reminder-triggered correctly routed"}
2025-03-17T04:00:00.198333+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8fcf7772-e019-4725-a77e-724a972a256e"}
2025-03-17T04:00:00.198508+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-reminder-triggered' for consumer group='icc-reporting'"}
2025-03-17T04:00:00.198544+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T04:00:00.189Z\",\"incident\":{\"id\":\"8fcf7772-e019-4725-a77e-724a972a256e\"},\"eventObj\":{\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"reminderId\":\"6b146ee2-81a9-449b-8d58-d72a8967f969\",\"message\":\"If the Incident is invalid, close using /icc-incident quickclose in slack or the overflow menu in the Web UI\",\"additionalInfo\":{\"rule\":{\"name\":\"Low Severity Incident Reminders\",\"type\":\"team\"},\"status\":\"InProgress\"},\"reminderData\":{\"id\":\"6b146ee2-81a9-449b-8d58-d72a8967f969\",\"type\":\"REMINDER\",\"active\":true,\"message\":\"If the Incident is invalid, close using /icc-incident quickclose in slack or the overflow menu in the Web UI\",\"audience\":[],\"incidentId\":\"8fcf7772-e019-4725-a77e-724a972a256e\",\"offsetUnit\":\"m\",\"recurrence\":50,\"triggersAt\":\"2025-03-17T10:51:49.681Z\",\"isRecurring\":true,\"occurrences\":[{\"oid\":\"e37586ae-a8eb-40a7-b585-33b87c9facf3\",\"completed\":false,\"triggeredAt\":\"2025-03-15T06:51:49.767Z\"},{\"oid\":\"2372dadc-05d8-477d-9ec4-f6ecfbdf8280\",\"completed\":false,\"triggeredAt\":\"2025-03-15T08:00:00.104Z\"},{\"oid\":\"fcf49721-2859-4bb1-b936-6a36fa311737\",\"completed\":false,\"triggeredAt\":\"2025-03-15T12:00:00.190Z\"},{\"oid\":\"8ca45c1f-2f08-4c50-82c8-329d541e934d\",\"completed\":false,\"triggeredAt\":\"2025-03-15T16:00:00.151Z\"},{\"oid\":\"b3c5beb8-6f99-46e2-9ba9-ff486120e1e1\",\"completed\":false,\"triggeredAt\":\"2025-03-15T20:00:00.184Z\"},{\"oid\":\"d9c903a7-d6b6-4690-9b96-83bc50bc6788\",\"completed\":false,\"triggeredAt\":\"2025-03-16T00:00:00.127Z\"},{\"oid\":\"d5e17c17-b87c-46bc-81d1-67dcfdc231d2\",\"completed\":false,\"triggeredAt\":\"2025-03-16T04:00:00.131Z\"},{\"oid\":\"004e277f-69b3-4312-891b-7b749f2ea14b\",\"completed\":false,\"triggeredAt\":\"2025-03-16T08:00:00.168Z\"},{\"oid\":\"cc16a56c-2a50-40e0-adb1-7ad00bce1cbe\",\"completed\":false,\"triggeredAt\":\"2025-03-16T12:00:00.101Z\"},{\"oid\":\"8305e75c-4f47-451d-8003-0cf3ec3c248b\",\"completed\":false,\"triggeredAt\":\"2025-03-16T16:00:00.104Z\"},{\"oid\":\"8d3ff226-f126-429d-ac0b-56c42e3d74d7\",\"completed\":false,\"triggeredAt\":\"2025-03-16T20:00:00.166Z\"},{\"oid\":\"2b277be2-7ee3-43a2-a714-d8c6d9255a29\",\"completed\":false,\"triggeredAt\":\"2025-03-17T00:00:00.177Z\"},{\"oid\":\"65b09dde-5db8-4fdc-928e-b5120109e27d\",\"triggeredAt\":\"2025-03-17T04:00:00.169Z\",\"completed\":false}],\"offsetValue\":5,\"frequencyUnit\":\"h\",\"additionalInfo\":{\"rule\":{\"name\":\"Low Severity Incident Reminders\",\"type\":\"team\"},\"status\":\"InProgress\"},\"frequencyValue\":4,\"currentOccurrence\":13,\"currentOccurrenceId\":\"65b09dde-5db8-4fdc-928e-b5120109e27d\",\"inheritedFromTeamConfig\":true},\"slackChannelId\":\"C08J5R8VCGH\",\"nextDueAt\":\"2025-03-17T10:51:49.681Z\"}}"}
2025-03-17T04:00:00.198587+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T04:00:00.198619+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T04:00:00.198739+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:00:00.198742+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8fcf7772-e019-4725-a77e-724a972a256e - icc-reminder-triggered"}
2025-03-17T04:00:00.198743+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-reminder-triggered is not consumed by icc-reporting."}
2025-03-17T04:00:00.198773+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184000198,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8fcf7772-e019-4725-a77e-724a972a256e - icc-reminder-triggered correctly routed"}
2025-03-17T04:04:13.582700+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184253582,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T04:04:13.583471+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184253583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T04:14:04.189475+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742184844189,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891501568,\"uptime\":2004739.57,\"totalMem\":4018151424,\"loadavg\":[0.04,0.06,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24110680,\"nice\":16950,\"sys\":3458650,\"idle\":1969741370,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23829040,\"nice\":16410,\"sys\":3474660,\"idle\":1969964710,\"irq\":0}}],\"eventLoopStats\":{\"min\":29984,\"max\":30045,\"num\":2,\"sum\":60029}}}"}
2025-03-17T04:14:04.189540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742184844189,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:17:04.273459+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742185024273,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2881667072,\"uptime\":2004919.66,\"totalMem\":4018151424,\"loadavg\":[0.03,0.08,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24112160,\"nice\":16950,\"sys\":3458920,\"idle\":1969918600,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23830410,\"nice\":16410,\"sys\":3475000,\"idle\":1970142210,\"irq\":0}}],\"eventLoopStats\":{\"min\":29989,\"max\":30038,\"num\":2,\"sum\":60027}}}"}
2025-03-17T04:17:04.273516+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742185024273,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:18:04.301462+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742185084301,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2880749568,\"uptime\":2004979.68,\"totalMem\":4018151424,\"loadavg\":[0.01,0.06,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24112720,\"nice\":16950,\"sys\":3458960,\"idle\":1969977930,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23831130,\"nice\":16410,\"sys\":3475110,\"idle\":1970201440,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30025,\"num\":3,\"sum\":60027}}}"}
2025-03-17T04:18:04.301539+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742185084301,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:21:04.369443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742185264369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2879959040,\"uptime\":2005159.75,\"totalMem\":4018151424,\"loadavg\":[0.09,0.07,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24114340,\"nice\":16950,\"sys\":3459210,\"idle\":1970155620,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23832780,\"nice\":16410,\"sys\":3475320,\"idle\":1970379250,\"irq\":0}}],\"eventLoopStats\":{\"min\":17,\"max\":30008,\"num\":3,\"sum\":60029}}}"}
2025-03-17T04:21:04.369519+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742185264369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:29:04.573423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742185744573,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2879361024,\"uptime\":2005639.96,\"totalMem\":4018151424,\"loadavg\":[0.03,0.04,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24118240,\"nice\":16950,\"sys\":3459780,\"idle\":1970629450,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23836620,\"nice\":16410,\"sys\":3475910,\"idle\":1970853560,\"irq\":0}}],\"eventLoopStats\":{\"min\":26,\"max\":30029,\"num\":3,\"sum\":60024}}}"}
2025-03-17T04:29:04.573487+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742185744573,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:34:19.255466+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186059255,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T04:34:19.256335+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186059256,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T04:35:04.705504+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742186104705,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2879483904,\"uptime\":2006000.09,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24120490,\"nice\":16950,\"sys\":3460250,\"idle\":1970984800,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23839190,\"nice\":16410,\"sys\":3476280,\"idle\":1971209320,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29983,\"num\":13,\"sum\":60006}}}"}
2025-03-17T04:35:04.705569+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186104705,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:44:03.459929+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186643459,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T04:44:03.460723+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186643460,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T04:45:04.873422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742186704873,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2880286720,\"uptime\":2006600.26,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24124950,\"nice\":16950,\"sys\":3460980,\"idle\":1971577060,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23843870,\"nice\":16410,\"sys\":3477140,\"idle\":1971802040,\"irq\":0}}],\"eventLoopStats\":{\"min\":52,\"max\":30002,\"num\":3,\"sum\":60007}}}"}
2025-03-17T04:45:04.873503+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186704873,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:47:04.917424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742186824917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2882514944,\"uptime\":2006720.3,\"totalMem\":4018151424,\"loadavg\":[0.04,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24126780,\"nice\":16950,\"sys\":3461200,\"idle\":1971695170,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23845250,\"nice\":16410,\"sys\":3477400,\"idle\":1971920210,\"irq\":0}}],\"eventLoopStats\":{\"min\":48,\"max\":30029,\"num\":3,\"sum\":60016}}}"}
2025-03-17T04:47:04.917549+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742186824917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:51:04.981390+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742187064981,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887974912,\"uptime\":2006960.37,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24128980,\"nice\":16950,\"sys\":3461550,\"idle\":1971932240,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23847470,\"nice\":16410,\"sys\":3477690,\"idle\":1972157160,\"irq\":0}}],\"eventLoopStats\":{\"min\":35,\"max\":30028,\"num\":3,\"sum\":60005}}}"}
2025-03-17T04:51:04.981514+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187064981,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:54:05.005380+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742187245005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888740864,\"uptime\":2007140.39,\"totalMem\":4018151424,\"loadavg\":[0.06,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24130510,\"nice\":16950,\"sys\":3461680,\"idle\":1972110060,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23849620,\"nice\":16410,\"sys\":3477910,\"idle\":1972334860,\"irq\":0}}],\"eventLoopStats\":{\"min\":50,\"max\":30028,\"num\":3,\"sum\":60002}}}"}
2025-03-17T04:54:05.005525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187245005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:54:05.434940+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187245434,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T04:54:05.435682+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187245435,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T04:55:05.005424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742187305005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887569408,\"uptime\":2007200.39,\"totalMem\":4018151424,\"loadavg\":[0.08,0.04,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24131000,\"nice\":16950,\"sys\":3461760,\"idle\":1972169310,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23850070,\"nice\":16410,\"sys\":3477990,\"idle\":1972394110,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29917,\"num\":13,\"sum\":59996}}}"}
2025-03-17T04:55:05.005520+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187305005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T04:57:05.037455+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742187425037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886664192,\"uptime\":2007320.42,\"totalMem\":4018151424,\"loadavg\":[0.11,0.07,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24131740,\"nice\":16950,\"sys\":3461890,\"idle\":1972287850,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23850850,\"nice\":16410,\"sys\":3478140,\"idle\":1972512550,\"irq\":0}}],\"eventLoopStats\":{\"min\":81,\"max\":30029,\"num\":3,\"sum\":60002}}}"}
2025-03-17T04:57:05.037616+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187425037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:04:07.290710+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187847290,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T05:04:07.291514+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742187847291,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T05:07:05.273420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188025273,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884210688,\"uptime\":2007920.66,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24135980,\"nice\":16950,\"sys\":3462660,\"idle\":1972880800,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23855750,\"nice\":16410,\"sys\":3478830,\"idle\":1973104940,\"irq\":0}}],\"eventLoopStats\":{\"min\":115,\"max\":30026,\"num\":3,\"sum\":60027}}}"}
2025-03-17T05:07:05.273481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188025273,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:14:05.429483+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188445429,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884390912,\"uptime\":2008340.81,\"totalMem\":4018151424,\"loadavg\":[0,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24139520,\"nice\":16950,\"sys\":3463230,\"idle\":1973295530,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23858640,\"nice\":16410,\"sys\":3479420,\"idle\":1973519770,\"irq\":0}}],\"eventLoopStats\":{\"min\":97,\"max\":30009,\"num\":3,\"sum\":60028}}}"}
2025-03-17T05:14:05.429537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188445429,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:14:39.922048+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188479921,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T05:14:39.922255+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T05:14:39.922301+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T05:14:39.913Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"9cdcd391-54e4-4118-a421-054cdb4af8fc\",\"sequenceNumber\":7}}"}
2025-03-17T05:14:39.922344+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T05:14:39.922378+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T05:14:39.922528+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:14:39.922529+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response"}
2025-03-17T05:14:39.922532+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T05:14:39.922545+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188479922,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response correctly routed"}
2025-03-17T05:15:05.453410+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188505453,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884399104,\"uptime\":2008400.84,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24139970,\"nice\":16950,\"sys\":3463300,\"idle\":1973354750,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23859170,\"nice\":16410,\"sys\":3479540,\"idle\":1973579030,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":25511,\"num\":20,\"sum\":60019}}}"}
2025-03-17T05:15:05.453511+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188505453,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:16:05.485439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188565485,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884169728,\"uptime\":2008460.87,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24140610,\"nice\":16950,\"sys\":3463430,\"idle\":1973413670,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23859790,\"nice\":16410,\"sys\":3479700,\"idle\":1973637870,\"irq\":0}}],\"eventLoopStats\":{\"min\":57,\"max\":30011,\"num\":3,\"sum\":60027}}}"}
2025-03-17T05:16:05.485492+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188565485,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:17:05.513404+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742188625513,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891141120,\"uptime\":2008520.9,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24141110,\"nice\":16950,\"sys\":3463560,\"idle\":1973472870,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23860320,\"nice\":16410,\"sys\":3479810,\"idle\":1973697150,\"irq\":0}}],\"eventLoopStats\":{\"min\":40,\"max\":30016,\"num\":3,\"sum\":60030}}}"}
2025-03-17T05:17:05.513537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742188625513,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:24:30.424601+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742189070424,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T05:24:30.425525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742189070425,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T05:28:05.717438+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742189285717,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2897391616,\"uptime\":2009181.1,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24146210,\"nice\":16950,\"sys\":3464510,\"idle\":1974124890,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23865930,\"nice\":16410,\"sys\":3480740,\"idle\":1974348680,\"irq\":0}}],\"eventLoopStats\":{\"min\":18,\"max\":30000,\"num\":3,\"sum\":60008}}}"}
2025-03-17T05:28:05.717539+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742189285717,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:37:05.885412+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742189825885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890424320,\"uptime\":2009721.27,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24150330,\"nice\":16950,\"sys\":3465230,\"idle\":1974658260,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23870890,\"nice\":16410,\"sys\":3481400,\"idle\":1974881810,\"irq\":0}}],\"eventLoopStats\":{\"min\":20,\"max\":30013,\"num\":3,\"sum\":60028}}}"}
2025-03-17T05:37:05.885531+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742189825885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:38:05.913420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742189885913,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890461184,\"uptime\":2009781.3,\"totalMem\":4018151424,\"loadavg\":[0.17,0.05,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24150850,\"nice\":16950,\"sys\":3465290,\"idle\":1974717590,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23871510,\"nice\":16410,\"sys\":3481480,\"idle\":1974941070,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30009,\"num\":3,\"sum\":60029}}}"}
2025-03-17T05:38:05.913472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742189885913,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:40:05.949418+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742190005949,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890469376,\"uptime\":2009901.33,\"totalMem\":4018151424,\"loadavg\":[0.02,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24151830,\"nice\":16950,\"sys\":3465460,\"idle\":1974836140,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23872820,\"nice\":16410,\"sys\":3481650,\"idle\":1975059530,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30005,\"num\":13,\"sum\":60019}}}"}
2025-03-17T05:40:05.949476+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190005949,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:49:35.465882+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190575465,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T05:49:35.466658+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190575466,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T05:51:06.165476+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742190666165,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889871360,\"uptime\":2010561.55,\"totalMem\":4018151424,\"loadavg\":[0.04,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24156540,\"nice\":16950,\"sys\":3466450,\"idle\":1975487880,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23879450,\"nice\":16410,\"sys\":3482760,\"idle\":1975710510,\"irq\":0}}],\"eventLoopStats\":{\"min\":19,\"max\":30027,\"num\":3,\"sum\":60018}}}"}
2025-03-17T05:51:06.165611+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190666165,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T05:54:36.340569+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190876340,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T05:54:36.341365+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742190876341,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T05:58:06.297615+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191086297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891128832,\"uptime\":2010981.68,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24159700,\"nice\":16950,\"sys\":3467130,\"idle\":1975902650,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23882670,\"nice\":16410,\"sys\":3483390,\"idle\":1976125240,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30027,\"num\":17,\"sum\":59995}}}"}
2025-03-17T05:58:06.297736+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191086297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:03:06.365458+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191386365,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890137600,\"uptime\":2011281.75,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24161850,\"nice\":16950,\"sys\":3467550,\"idle\":1976199030,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23884900,\"nice\":16410,\"sys\":3483740,\"idle\":1976421370,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30027,\"num\":13,\"sum\":59995}}}"}
2025-03-17T06:03:06.365520+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191386365,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:09:06.465394+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191746465,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889338880,\"uptime\":2011641.85,\"totalMem\":4018151424,\"loadavg\":[0.11,0.05,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24164390,\"nice\":16950,\"sys\":3467990,\"idle\":1976554440,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23887680,\"nice\":16410,\"sys\":3484130,\"idle\":1976776970,\"irq\":0}}],\"eventLoopStats\":{\"min\":116,\"max\":30030,\"num\":3,\"sum\":60003}}}"}
2025-03-17T06:09:06.465543+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191746465,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:12:58.623494+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T06:12:58.623689+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-page-request' for consumer group='icc-reporting'"}
2025-03-17T06:12:58.623756+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:12:58.209Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"ok\":true,\"id\":\"35c1378a-59da-4c92-837b-a74795efb907\",\"serviceId\":\"PXJUOWJ\",\"customMessage\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: Customers seeing latency on OMNI chat - single pod\\nSeverity: Sev2\\nSlack Channel: #icc-73095026\\nBridge URL: https://meet.google.com/hbh-zkju-hsu\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\\nAdditional Details: \",\"incidentSubject\":\"Customers seeing latency on OMNI chat - single pod\",\"incidentDescription\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\"},\"requestId\":\"a4db3b5d-7b9b-3bf4-2142-056cbbb2cd5a\",\"client\":\"icc-slack-app\"}"}
2025-03-17T06:12:58.623758+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T06:12:58.623800+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T06:12:58.623926+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:12:58.623927+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-page-request"}
2025-03-17T06:12:58.623929+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-page-request is not consumed by icc-reporting."}
2025-03-17T06:12:58.623955+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742191978623,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-page-request correctly routed"}
2025-03-17T06:14:06.489470+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192046489,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894921728,\"uptime\":2011941.87,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24167140,\"nice\":16990,\"sys\":3468430,\"idle\":1976850190,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23890070,\"nice\":16540,\"sys\":3484550,\"idle\":1977072550,\"irq\":0}}],\"eventLoopStats\":{\"min\":215,\"max\":30009,\"num\":3,\"sum\":60018}}}"}
2025-03-17T06:14:06.489530+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192046489,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:16:06.537433+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192166537,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894217216,\"uptime\":2012061.92,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24167870,\"nice\":16990,\"sys\":3468590,\"idle\":1976968840,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23891020,\"nice\":16540,\"sys\":3484710,\"idle\":1977190960,\"irq\":0}}],\"eventLoopStats\":{\"min\":207,\"max\":30006,\"num\":3,\"sum\":60026}}}"}
2025-03-17T06:16:06.537487+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192166537,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:17:06.557454+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192226557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894065664,\"uptime\":2012121.94,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24168520,\"nice\":16990,\"sys\":3468740,\"idle\":1977027670,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23891670,\"nice\":16540,\"sys\":3484830,\"idle\":1977249880,\"irq\":0}}],\"eventLoopStats\":{\"min\":186,\"max\":30030,\"num\":3,\"sum\":60019}}}"}
2025-03-17T06:17:06.557520+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192226557,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:27:18.577291+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838577,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T06:27:18.578551+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-request' for consumer group='icc-reporting'"}
2025-03-17T06:27:18.578716+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:27:18.409Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"bf978938-ae01-4224-b772-cf0c107893d2\",\"allInternalCommsReport\":[{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStart
2025-03-17T06:27:18.578716+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] Time\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"f6cfdd0a-1027-7931-b8aa-b3ff9f13899f\",\"client\":\"icc-slack-app\"}"}
2025-03-17T06:27:18.578717+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T06:27:18.578741+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T06:27:18.578861+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:27:18.578861+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-request"}
2025-03-17T06:27:18.578862+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-internal-comms-request is now being processed."}
2025-03-17T06:27:18.579035+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838579,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId QWCAAANuCF5 and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,116,101,114,110,97,108,45,99,111,109,109,115,45,114,101,113,117,101,115,116]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,54,58,50,55,58,49,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T06:27:18.409Z\\\",\\\"user\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J0HPF5H9\\\",\\\"name\\\":\\\"icc-73095026\\\"},\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"description\\\":\\\"CP SRE Incident\\\\n\\\\nCase created using .case by @Priyanka Sharma in #sr for NA208\\\",\\\"detectedTime\\\":\\\"2025-03-17T00:25:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\\\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\\\n\\\"},\\\"roles\\\":[{\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"id\\\":\\\"80955e31-ab05-4427-add9-3a093398312f\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\\\",\\\"email\\\":\\\"psharma2@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Priyanka Sharma\\\",\\\"firstName\\\":\\\"Priyanka\\\",\\\"lastName\\\":\\\"Sharma\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005EE000001oiQHYAY\\\",\\\"slackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"createdBy\\\":\\\"icc@aloha.sso\\\",\\\"modifiedBy\\\":\\\"psharma2@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-03-03T06:58:10.326Z\\\",\\\"modifiedAt\\\":\\\"2022-03-03T06:59:00.333Z\\\"},\\\"externalGusId\\\":null}],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73095026\\\",\\\"caseRecordId\\\":\\\"500EE00001Ty0FFYAZ\\\",\\\"servicesImpacted\\\":[\\\"Core 1P\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/M8S4Ap02KbIu\\\",\\\"engagedAt\\\":\\\"2025-03-17T00:26:23.518Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T00:38:13.514Z\\\"},\\\"eventObj\\\":{\\\"internalComms\\\":{\\\"sequenceNumber\\\":9,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"6 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\\\n\\\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\\\",\\\"actions\\\":\\\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\\\n\\\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\\\"},\\\"sentAt\\\":\\\"2025-03-17T06:27:18.054Z\\\",\\\"swimlaneUpdates\\\":[],\\\"slackChannelIds\\\":[\\\"C05U9CBHX8D\\\",\\\"C05T2A3C319\\\"],\\\"emailAddresses\\\":[\\\"gsm@salesforce.com\\\",\\\"sitereliability@salesforce.com\\\",\\\"techcustomerescalation@salesforce.com\\\"]},\\\"internalCommsId\\\":\\\"bf978938-ae01-4224-b772-cf0c107893d2\\\",\\\"allInternalCommsReport\\\":[{\\\"sequenceNumber\\\":9,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"6 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\\\n\\\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\\\",\\\"actions\\\":\\\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\\\n\\\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\\\"},\\\"sentAt\\\":\\\"2025-03-17T06:27:18.054Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":8,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"5 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"Omnichat team advises that they're seeing a customer \\\\\\\"Schneider electric\\\\\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\\\n\\\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\\\",\\\"needs\\\":\\\"Support will create an outbound case for Schneider
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] electric.\\\"},\\\"sentAt\\\":\\\"2025-03-17T05:55:49.276Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":7,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"5 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\\\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\\\nInvestigation and mitigation steps continue to happen however.\\\"},\\\"sentAt\\\":\\\"2025-03-17T05:14:27.661Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":6,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"4 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\\\",\\\"actions\\\":\\\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\\\"},\\\"sentAt\\\":\\\"2025-03-17T04:39:02.111Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":5,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\\\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\\\"},\\\"sentAt\\\":\\\"2025-03-17T03:54:10.722Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":4,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\\\"},\\\"sentAt\\\":\\\"2025-03-17T03:06:01.346Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":3,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"112 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\\\",\\\"actions\\\":\\\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\\\"},\\\"sentAt\\\":\\\"2025-03-17T02:17:40.848Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":2,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"60 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\\\",\\\"actions\\\":\\\"Omni team is investigating the causes for latency.\\\\nHeap dumps are being collected for the hosts to analyse.\\\\nRyan who actually owns the service is paged to investigate further\\\"},\\\"sentAt\\\":\\\"2025-03-17T01:25:49.250Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":1,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"14 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\\\",\\\"actions\\\":\\\"Investigating the issue and impact radius is under discussion\\\"},\\\"sentAt\\\":\\\"2025-03-17T00:40:19.849Z\\\",\\\"swimlaneUpdates\\\":[]}]},\\\"requestId\\\":\\\"f6cfdd0a-1027-7931-b8aa-b3ff9f13899f\\\",\\\"client\\\":\\\"icc-slack-app\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T06:27:18.409Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] 24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"bf978938-ae01-4224-b772-cf0c107893d2\",\"allInternalCommsReport\":[{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is re
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] sumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"f6cfdd0a-1027-7931-b8aa-b3ff9f13899f\",\"client\":\"icc-slack-app\"},\"encoder\":\"json\",\"eventType\":\"icc-internal-comms-request\",\"payload\":{\"createdAt\":\"2025-03-17T06:27:18.409Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlack
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] UserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"bf978938-ae01-4224-b772-cf0c107893d2\",\"allInternalCommsReport\":[{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"condi
2025-03-17T06:27:18.579782+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] tions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"f6cfdd0a-1027-7931-b8aa-b3ff9f13899f\",\"client\":\"icc-slack-app\"}}"}
2025-03-17T06:27:18.579783+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838579,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: QWCAAANuCF5"}
2025-03-17T06:27:18.579783+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838579,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-internal-comms-request and docId QWCAAANuCF5"}
2025-03-17T06:27:18.579784+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192838579,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleInternalCommsRequest for id 13054fa1-b6e2-45d6-8afe-0c7523384b29 and docId QWCAAANuCF5"}
2025-03-17T06:27:18.580272+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742192838580,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"this.update.content.replace is not a function","stack":"TypeError: this.update.content.replace is not a function\n    at UpdateInfo.getContentWithNewlines (/app/build/quip/templates/update.info.js:21:36)\n    at Context.lookup (/app/node_modules/mustache/mustache.js:477:21)\n    at Writer.unescapedValue (/app/node_modules/mustache/mustache.js:664:25)\n    at Writer.renderTokens (/app/node_modules/mustache/mustache.js:580:45)\n    at Writer.render (/app/node_modules/mustache/mustache.js:556:17)\n    at Object.render (/app/node_modules/mustache/mustache.js:758:26)\n    at QuipService.addUpdate (/app/build/quip/quip.service.js:98:41)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at async ExportService.addUpdate (/app/build/service/export.service.js:141:16)\n    at async ExportService.addInternalCommsUpdate (/app/build/service/export.service.js:93:16)"},"msg":"[Error] message not processed"}
2025-03-17T06:27:30.004775+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192850004,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T06:27:30.004980+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192850004,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T06:27:30.005030+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192850004,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:27:29.995Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"bf978938-ae01-4224-b772-cf0c107893d2\",\"sequenceNumber\":9}}"}
2025-03-17T06:27:30.005060+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T06:27:30.005131+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T06:27:30.005258+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:27:30.005258+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response"}
2025-03-17T06:27:30.005306+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T06:27:30.005306+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192850005,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response correctly routed"}
2025-03-17T06:28:06.741398+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192886741,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888781824,\"uptime\":2012782.12,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24175830,\"nice\":16990,\"sys\":3469920,\"idle\":1977678000,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23896890,\"nice\":16540,\"sys\":3485910,\"idle\":1977901350,\"irq\":0}}],\"eventLoopStats\":{\"min\":1,\"max\":29816,\"num\":10,\"sum\":60004}}}"}
2025-03-17T06:28:06.741554+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192886741,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:29:48.297698+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"request\",\"data\":{\"method\":\"quip\",\"statusCode\":0,\"message\":\"\"}}"}
2025-03-17T06:29:48.297837+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:29:48.298068+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988298,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"QuipService.createDocument for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 created with URL https://salesforce.quip.com/ZHQDAc9AH2Vv"}
2025-03-17T06:29:48.298306+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988298,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.setValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: bJBAAAadCyL"}
2025-03-17T06:29:48.299556+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988299,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info, info: {\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseId\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"slackChannelId\":\"C08HYDW3893\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"severity\":\"Sev3\",\"status\":\"New\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T06:29:41.802Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Services Impacted not found\",\"roles\":{},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"caseNumber\":\"Case Number not found\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T06:29:48.299893+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988299,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"produceEvent with link https://salesforce.quip.com/ZHQDAc9AH2Vv for event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and case "}
2025-03-17T06:29:48.299975+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988299,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricRequestExecute] - icc-report-created - undefined"}
2025-03-17T06:29:48.300027+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988299,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T06:29:48.300083+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T06:29:48.300203+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:29:48.300469+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key '4a6faf32-c583-4c0e-b53f-ee1b00fd55c6' to topic 'incident': {\"createdAt\":\"2025-03-17T06:29:48.299Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:34.000Z\",\"severity\":\"Sev3\",\"status\":\"New\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T06:29:41.802Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\"}}"}
2025-03-17T06:29:48.300544+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key '4a6faf32-c583-4c0e-b53f-ee1b00fd55c6' to Kafka topic 'incident'"}
2025-03-17T06:29:48.300547+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricDurationTime] - icc-report-created - undefined"}
2025-03-17T06:29:48.300567+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.2260509990155697\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T06:29:48.300624+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.2260509990155697\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T06:29:48.300653+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:29:48.300804+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192988300,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-created correctly routed"}
2025-03-17T06:29:51.429019+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192991428,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-gus-response correctly routed"}
2025-03-17T06:29:58.329672+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:29:58.329753+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-ctc-lock-response' for consumer group='icc-reporting'"}
2025-03-17T06:29:58.329754+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"locks\":[{\"id\":\"6468ef25-d0e7-4f36-af75-653590f9028a\",\"status\":\"active\",\"externalId\":\"a8LEE00001BcYpw2AF\",\"incidentToCiId\":\"7055addb-45b2-4ea7-9b87-87371b687582\"}],\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"}"}
2025-03-17T06:29:58.329768+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T06:29:58.329834+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T06:29:58.329926+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192998329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:29:58.330179+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742192998330,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T06:29:58.565437+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742192998565,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info, info: {\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseId\":\"500EE00001TybJdYAJ\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"slackChannelId\":\"C08HYDW3893\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"severity\":\"Sev3\",\"status\":\"InProgress\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T06:29:41.802Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Services Impacted not found\",\"roles\":{\"Incident Commander\":\"Anna Marie Kaibigan\"},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"caseNumber\":\"73102086\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T06:29:58.566566+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742192998566,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-updated correctly routed"}
2025-03-17T06:30:06.741414+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193006741,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887749632,\"uptime\":2012902.12,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24177320,\"nice\":16990,\"sys\":3470070,\"idle\":1977796320,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23898010,\"nice\":16540,\"sys\":3486090,\"idle\":1978019840,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30005,\"num\":102,\"sum\":59956}}}"}
2025-03-17T06:30:06.741544+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193006741,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:30:39.946448+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193039946,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-bridge-created correctly routed"}
2025-03-17T06:30:55.885503+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:30:55.885667+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T06:30:55.885677+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:30:55.675Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T06:30:55.675Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}}"}
2025-03-17T06:30:55.885677+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T06:30:55.885733+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T06:30:55.885860+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:30:55.885861+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-gus-response"}
2025-03-17T06:30:55.885861+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T06:30:55.886039+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T06:30:55.886402+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,54,58,51,48,58,53,53,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T06:30:55.675Z\\\",\\\"user\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_updated\\\",\\\"timestamp\\\":\\\"2025-03-17T06:30:55.675Z\\\",\\\"recordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"incidentId\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"caseNumber\\\":\\\"73102086\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T06:30:55.675Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T06:30:55.675Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T06:30:55.675Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T06:30:55.675Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}}}"}
2025-03-17T06:30:55.886419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T06:30:55.886419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-gus-response and docId bJBAAAadCyL"}
2025-03-17T06:30:55.886420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleGusExport for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T06:30:55.886424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6, docId bJBAAAadCyL, event {\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T06:30:55.675Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"}"}
2025-03-17T06:30:55.886536+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-gus-casenumber - value: [object Promise]"}
2025-03-17T06:30:55.886836+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6, docId bJBAAAadCyL, caseNumber 73102086"}
2025-03-17T06:30:55.886837+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193055886,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-gus-response correctly routed"}
2025-03-17T06:31:20.183504+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193080183,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-page-response correctly routed"}
2025-03-17T06:39:06.857428+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193546857,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890514432,\"uptime\":2013442.24,\"totalMem\":4018151424,\"loadavg\":[0.03,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24181200,\"nice\":16990,\"sys\":3470670,\"idle\":1978329520,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23901810,\"nice\":16540,\"sys\":3486910,\"idle\":1978553050,\"irq\":0}}],\"eventLoopStats\":{\"min\":249,\"max\":30012,\"num\":3,\"sum\":60019}}}"}
2025-03-17T06:39:06.857508+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193546857,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:41:06.913416+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193666913,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890047488,\"uptime\":2013562.3,\"totalMem\":4018151424,\"loadavg\":[0.03,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24182710,\"nice\":16990,\"sys\":3470900,\"idle\":1978447960,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23902990,\"nice\":16540,\"sys\":3487090,\"idle\":1978671640,\"irq\":0}}],\"eventLoopStats\":{\"min\":244,\"max\":30003,\"num\":3,\"sum\":60026}}}"}
2025-03-17T06:41:06.913561+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193666913,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:41:21.488586+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193681488,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T06:41:21.489525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193681489,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T06:42:06.921431+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193726921,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890080256,\"uptime\":2013622.31,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24183400,\"nice\":16990,\"sys\":3470970,\"idle\":1978507160,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23903500,\"nice\":16540,\"sys\":3487160,\"idle\":1978730920,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29773,\"num\":13,\"sum\":60005}}}"}
2025-03-17T06:42:06.921556+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193726921,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:16.567725+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796567,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:16.567941+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796567,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T06:43:16.568016+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796567,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:16.557Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev3\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T06:29:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:29:41.802Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:30:55.675Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"t14yjm-xr7y9g-a14i4a-kl9ofs-1742193796142\",\"client\":\"icc-ui\"}"}
2025-03-17T06:43:16.568060+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T06:43:16.568101+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T06:43:16.568255+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:16.568256+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-usd-resource-deleted"}
2025-03-17T06:43:16.568257+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T06:43:16.568260+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796568,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-usd-resource-deleted correctly routed"}
2025-03-17T06:43:16.588956+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796588,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:16.589194+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-created' for consumer group='icc-reporting'"}
2025-03-17T06:43:16.589263+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:16.579Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev3\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T06:29:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:29:41.802Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:30:55.675Z\"},\"eventObj\":[],\"requestId\":\"t14yjm-xr7y9g-a14i4a-kl9ofs-1742193796142\",\"client\":\"icc-ui\"}"}
2025-03-17T06:43:16.589272+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T06:43:16.589309+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T06:43:16.589421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:16.589421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-usd-resource-created"}
2025-03-17T06:43:16.589422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-created is not consumed by icc-reporting."}
2025-03-17T06:43:16.589451+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-usd-resource-created correctly routed"}
2025-03-17T06:43:16.637563+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:16.637623+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-response' for consumer group='icc-reporting'"}
2025-03-17T06:43:16.637681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:16.589Z\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"responses\":[]}"}
2025-03-17T06:43:16.637682+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T06:43:16.637727+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T06:43:16.637829+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:16.638058+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742193796637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T06:43:17.545676+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:17.545815+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-to-cis-response' for consumer group='icc-reporting'"}
2025-03-17T06:43:17.545827+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:17.495Z\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"responses\":[{\"response\":{\"id\":\"aCqEE000000C10L0AS\",\"success\":true,\"errors\":[]},\"configurationItemId\":\"a7JEE00000J7QcN2AV\"}]}"}
2025-03-17T06:43:17.545831+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-response\"}}"}
2025-03-17T06:43:17.545900+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-response\"}}"}
2025-03-17T06:43:17.545983+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193797545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:17.546218+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742193797546,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T06:43:17.578453+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:17.578579+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-to-cis-export-success' for consumer group='icc-reporting'"}
2025-03-17T06:43:17.578616+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:17.495Z\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"responses\":[{\"response\":{\"id\":\"aCqEE000000C10L0AS\",\"success\":true,\"errors\":[]},\"configurationItemId\":\"a7JEE00000J7QcN2AV\"}],\"cis\":[{\"id\":\"7055addb-45b2-4ea7-9b87-87371b687582\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"createLock\":true,\"sfid\":\"aCqEE000000C10L0AS\",\"configurationItem\":{\"id\":\"a7JEE00000J7QcN2AV\",\"name\":\"swe96s\",\"type\":\"Hyperforce Cell\",\"path\":\"Salesforce.Hyperforce.commercial1.aws-prod16-eunorth1.core1.swe96s\",\"parentId\":\"a7JEE000005dCCv2AM\",\"status\":null,\"childrenCount\":-1},\"lock\":{\"createdAt\":\"2025-03-17T06:29:55.674Z\",\"modifiedAt\":\"2025-03-17T06:29:58.331Z\",\"id\":\"6468ef25-d0e7-4f36-af75-653590f9028a\",\"status\":\"active\",\"description\":\"Lock to be created for Incident: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"externalId\":\"a8LEE00001BcYpw2AF\"}}]}"}
2025-03-17T06:43:17.578632+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-export-success\"}}"}
2025-03-17T06:43:17.578694+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-export-success\"}}"}
2025-03-17T06:43:17.578801+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:17.579000+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742193797578,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T06:43:20.147012+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193800146,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info, info: {\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseId\":\"500EE00001TybJdYAJ\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"slackChannelId\":\"C08HYDW3893\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"impact\":\"CurrentlyUnderInvestigation\",\"impactStart\":\"2025-03-17T04:14:00.000Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Core Live on Falcon\",\"roles\":{\"Incident Commander\":\"Anna Marie Kaibigan\"},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"caseNumber\":\"73102086\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T06:43:20.148097+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193800148,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-updated correctly routed"}
2025-03-17T06:43:38.583167+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:38.583536+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-roles-updated' for consumer group='icc-reporting'"}
2025-03-17T06:43:38.583641+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:38.093Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:43:22.029Z\"},\"eventObj\":{\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}]},\"changes\":{\"IncidentCommunicationsOfficer\":[null,{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"}]},\"requestId\":\"0d0c9393-4c8e-3974-416d-3cfaa44766ee\",\"client\":\"icc-slack-app\"}"}
2025-03-17T06:43:38.583643+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-roles-updated\"}}"}
2025-03-17T06:43:38.583701+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-roles-updated\"}}"}
2025-03-17T06:43:38.584324+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:38.584326+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-roles-updated"}
2025-03-17T06:43:38.584327+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-roles-updated is not consumed by icc-reporting."}
2025-03-17T06:43:38.584328+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193818583,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-roles-updated correctly routed"}
2025-03-17T06:43:47.885540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:47.885589+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='bullmq-job-completed' for consumer group='icc-reporting'"}
2025-03-17T06:43:47.885659+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:47.834Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"},\"eventObj\":{\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"reminderId\":\"a43e82cb-9468-42bc-bbcb-1b91e17b4488\",\"message\":\"Enable Recording AND Transcription for the Google Meet\",\"additionalInfo\":{\"rule\":{\"name\":\"Reminder: Enable Recording and Transcription for Incident Bridges\",\"type\":\"elevated_users\"},\"status\":\"InProgress\"}},\"client\":\"icc-rest\"}"}
2025-03-17T06:43:47.885660+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T06:43:47.885720+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T06:43:47.885827+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:47.885827+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - bullmq-job-completed"}
2025-03-17T06:43:47.885828+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type bullmq-job-completed is not consumed by icc-reporting."}
2025-03-17T06:43:47.885844+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827885,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - bullmq-job-completed correctly routed"}
2025-03-17T06:43:47.958079+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:43:47.958205+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-reminder-triggered' for consumer group='icc-reporting'"}
2025-03-17T06:43:47.958230+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:43:47.905Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"},\"eventObj\":{\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"reminderId\":\"a43e82cb-9468-42bc-bbcb-1b91e17b4488\",\"message\":\"Enable Recording AND Transcription for the Google Meet\",\"additionalInfo\":{\"rule\":{\"name\":\"Reminder: Enable Recording and Transcription for Incident Bridges\",\"type\":\"elevated_users\"},\"status\":\"InProgress\"},\"reminderData\":{\"id\":\"a43e82cb-9468-42bc-bbcb-1b91e17b4488\",\"type\":\"REMINDER\",\"active\":true,\"message\":\"Enable Recording AND Transcription for the Google Meet\",\"audience\":[],\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"offsetUnit\":\"m\",\"triggersAt\":\"2025-03-17T06:43:17.732Z\",\"isRecurring\":false,\"occurrences\":[],\"offsetValue\":0,\"additionalInfo\":{\"rule\":{\"name\":\"Reminder: Enable Recording and Transcription for Incident Bridges\",\"type\":\"elevated_users\"},\"status\":\"InProgress\"},\"currentOccurrence\":0,\"inheritedFromTeamConfig\":true},\"slackChannelId\":\"C08HYDW3893\"}}"}
2025-03-17T06:43:47.958250+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T06:43:47.958315+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T06:43:47.958419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:43:47.958420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-reminder-triggered"}
2025-03-17T06:43:47.958468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-reminder-triggered is not consumed by icc-reporting."}
2025-03-17T06:43:47.958470+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193827958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-reminder-triggered correctly routed"}
2025-03-17T06:45:06.981420+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742193906981,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2889216000,\"uptime\":2013802.36,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24184340,\"nice\":16990,\"sys\":3471230,\"idle\":1978685030,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23904800,\"nice\":16540,\"sys\":3487420,\"idle\":1978908520,\"irq\":0}}],\"eventLoopStats\":{\"min\":205,\"max\":30027,\"num\":3,\"sum\":60023}}}"}
2025-03-17T06:45:06.981500+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742193906981,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:49:07.045435+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194147045,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2899906560,\"uptime\":2014042.43,\"totalMem\":4018151424,\"loadavg\":[0.07,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24186480,\"nice\":16990,\"sys\":3471600,\"idle\":1978921620,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23906840,\"nice\":16540,\"sys\":3487690,\"idle\":1979145120,\"irq\":0}}],\"eventLoopStats\":{\"min\":207,\"max\":30027,\"num\":3,\"sum\":60018}}}"}
2025-03-17T06:49:07.045527+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194147045,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:50:07.065496+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194207065,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2899783680,\"uptime\":2014102.45,\"totalMem\":4018151424,\"loadavg\":[0.02,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24186940,\"nice\":16990,\"sys\":3471660,\"idle\":1978980840,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23907210,\"nice\":16540,\"sys\":3487780,\"idle\":1979204380,\"irq\":0}}],\"eventLoopStats\":{\"min\":215,\"max\":30021,\"num\":3,\"sum\":60018}}}"}
2025-03-17T06:50:07.065582+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194207065,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:56:07.177628+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194567177,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2899128320,\"uptime\":2014462.56,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24190250,\"nice\":16990,\"sys\":3472220,\"idle\":1979336280,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23910350,\"nice\":16540,\"sys\":3488400,\"idle\":1979559970,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29802,\"num\":13,\"sum\":60001}}}"}
2025-03-17T06:56:07.177634+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194567177,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:58:42.393881+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722393,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T06:58:42.394265+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-request' for consumer group='icc-reporting'"}
2025-03-17T06:58:42.394404+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T06:58:42.221Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:43:42.913Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"8e4b511a-e0c5-4e17-8887-bde179eec16f\",\"allInternalCommsReport\":[{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"8fd9b4bf-9a6d-6ee4-57e7-e7121c7f4633\",\"client\":\"icc-slack-app\"}"}
2025-03-17T06:58:42.394406+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T06:58:42.394429+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T06:58:42.394525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T06:58:42.394559+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-internal-comms-request"}
2025-03-17T06:58:42.394582+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-internal-comms-request is now being processed."}
2025-03-17T06:58:42.394731+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722394,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T06:58:42.395247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722395,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,116,101,114,110,97,108,45,99,111,109,109,115,45,114,101,113,117,101,115,116]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,54,58,53,56,58,52,50,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T06:58:42.221Z\\\",\\\"user\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"icc-73102086\\\"},\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"description\\\":\\\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\\\nDetection Time: 2025-03-17 06:13:20.000 \\\\nReason: 50% Orgs Delayed for 24h \\\\n\\\\n\\\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\\\",\\\"detectedTime\\\":\\\"2025-03-17T06:13:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\\\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\\\n\\\"},\\\"roles\\\":[{\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"id\\\":\\\"0ecd9085-134c-40b3-8327-b62637ef6fe0\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"firstName\\\":\\\"Anna\\\",\\\"lastName\\\":\\\"Marie\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"createdBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"modifiedBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-05T23:58:47.585Z\\\",\\\"modifiedAt\\\":\\\"2022-04-05T23:58:47.585Z\\\"},\\\"externalGusId\\\":null},{\\\"createdAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"id\\\":\\\"04d73ed0-dea3-4192-b83f-366a6de240a2\\\",\\\"name\\\":\\\"IncidentCommunicationsOfficer\\\",\\\"displayName\\\":\\\"Incident Communications Officer\\\",\\\"user\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"externalGusId\\\":null}],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73102086\\\",\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/ZHQDAc9AH2Vv\\\",\\\"engagedAt\\\":\\\"2025-03-17T06:29:55.775Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T06:43:42.913Z\\\"},\\\"eventObj\\\":{\\\"internalComms\\\":{\\\"sequenceNumber\\\":2,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\\\",\\\"sentAt\\\":\\\"2025-03-17T06:58:41.877Z\\\",\\\"swimlaneUpdates\\\":[],\\\"slackChannelIds\\\":[\\\"C05U9CBHX8D\\\",\\\"C05T2A3C319\\\"],\\\"emailAddresses\\\":[\\\"gsm@salesforce.com\\\",\\\"sitereliability@salesforce.com\\\",\\\"techcustomerescalation@salesforce.com\\\"]},\\\"internalCommsId\\\":\\\"8e4b511a-e0c5-4e17-8887-bde179eec16f\\\",\\\"allInternalCommsReport\\\":[{\\\"sequenceNumber\\\":2,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\\\",\\\"sentAt\\\":\\\"2025-03-17T06:58:41.877Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":1,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\
2025-03-17T06:58:42.395247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] "InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\\\",\\\"actions\\\":\\\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\\\"},\\\"sentAt\\\":\\\"2025-03-17T06:45:09.128Z\\\",\\\"swimlaneUpdates\\\":[]}]},\\\"requestId\\\":\\\"8fd9b4bf-9a6d-6ee4-57e7-e7121c7f4633\\\",\\\"client\\\":\\\"icc-slack-app\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T06:58:42.221Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:43:42.913Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"8e4b511a-e0c5-4e17-8887-bde179eec16f\",\"allInternalCommsReport\":[{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\"
2025-03-17T06:58:42.395247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ,\"swimlaneUpdates\":[]}]},\"requestId\":\"8fd9b4bf-9a6d-6ee4-57e7-e7121c7f4633\",\"client\":\"icc-slack-app\"},\"encoder\":\"json\",\"eventType\":\"icc-internal-comms-request\",\"payload\":{\"createdAt\":\"2025-03-17T06:58:42.221Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T06:43:42.913Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"8e4b511a-e0c5-4e17-8887-bde179eec16f\",\"allInternalCommsReport\":[{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"8fd9b4bf-9a6d-6ee4-57e7-e7121c7f4633\",\"client\":\"icc-slack-app\"}}"}
2025-03-17T06:58:42.395247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722395,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T06:58:42.395248+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722395,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-internal-comms-request and docId bJBAAAadCyL"}
2025-03-17T06:58:42.395249+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194722395,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleInternalCommsRequest for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T07:03:07.317443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742194987317,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2895347712,\"uptime\":2014882.7,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24193670,\"nice\":16990,\"sys\":3472730,\"idle\":1979750960,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23913360,\"nice\":16540,\"sys\":3488980,\"idle\":1979974860,\"irq\":0}}],\"eventLoopStats\":{\"min\":248,\"max\":30019,\"num\":3,\"sum\":60026}}}"}
2025-03-17T07:03:07.317499+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742194987317,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:07:07.417396+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195227417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893025280,\"uptime\":2015122.8,\"totalMem\":4018151424,\"loadavg\":[0.12,0.09,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24195560,\"nice\":16990,\"sys\":3472990,\"idle\":1979987970,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23914960,\"nice\":16540,\"sys\":3489260,\"idle\":1980211990,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30021,\"num\":13,\"sum\":60017}}}"}
2025-03-17T07:07:07.417510+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195227417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:07:18.639753+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195238639,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info, info: {\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseId\":\"500EE00001TybJdYAJ\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"slackChannelId\":\"C08HYDW3893\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T04:14:00.000Z\",\"impactEnd\":\"2025-03-17T07:03:00.000Z\",\"servicesImpacted\":\"Core Live on Falcon\",\"roles\":{},\"category\":\"Application\",\"rootCause\":\"Other\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"caseNumber\":\"73102086\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T07:07:18.640860+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195238640,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-resolved correctly routed"}
2025-03-17T07:07:18.836095+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195238835,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:07:18.836145+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-ctc-lock-response' for consumer group='icc-reporting'"}
2025-03-17T07:07:18.836146+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"locks\":[{\"id\":\"6468ef25-d0e7-4f36-af75-653590f9028a\",\"status\":\"inactive\",\"externalId\":\"a8LEE00001BcYpw2AF\",\"incidentToCiId\":\"7055addb-45b2-4ea7-9b87-87371b687582\"}],\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"}"}
2025-03-17T07:07:18.836169+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T07:07:18.836244+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T07:07:18.836355+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:07:18.836574+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742195238836,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T07:08:19.275057+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299274,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:08:19.275513+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-request' for consumer group='icc-reporting'"}
2025-03-17T07:08:19.275632+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:08:18.598Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"7ec916e0-1c07-43cb-9465-332d8575e78b\",\"allInternalCommsReport\":[{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"dae6dabe-464a-b80b-a7b7-2e47459983e6\",\"client\":\"icc-slack-app\"}"}
2025-03-17T07:08:19.275634+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T07:08:19.275696+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T07:08:19.276005+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:08:19.276006+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-internal-comms-request"}
2025-03-17T07:08:19.276007+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-internal-comms-request is now being processed."}
2025-03-17T07:08:19.276009+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T07:08:19.276537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299276,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,116,101,114,110,97,108,45,99,111,109,109,115,45,114,101,113,117,101,115,116]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,55,58,48,56,58,49,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T07:08:18.598Z\\\",\\\"user\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"icc-73102086\\\"},\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"description\\\":\\\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\\\nDetection Time: 2025-03-17 06:13:20.000 \\\\nReason: 50% Orgs Delayed for 24h \\\\n\\\\n\\\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\\\",\\\"detectedTime\\\":\\\"2025-03-17T06:13:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Resolved\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T07:03:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\\\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\\\n\\\"},\\\"roles\\\":[{\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"id\\\":\\\"0ecd9085-134c-40b3-8327-b62637ef6fe0\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"firstName\\\":\\\"Anna\\\",\\\"lastName\\\":\\\"Marie\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"createdBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"modifiedBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-05T23:58:47.585Z\\\",\\\"modifiedAt\\\":\\\"2022-04-05T23:58:47.585Z\\\"},\\\"externalGusId\\\":null},{\\\"createdAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T06:43:37.842Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"id\\\":\\\"04d73ed0-dea3-4192-b83f-366a6de240a2\\\",\\\"name\\\":\\\"IncidentCommunicationsOfficer\\\",\\\"displayName\\\":\\\"Incident Communications Officer\\\",\\\"user\\\":{\\\"id\\\":\\\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\\\",\\\"email\\\":\\\"tuannguyen@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Tuan Nguyen\\\",\\\"firstName\\\":\\\"Tuan\\\",\\\"lastName\\\":\\\"Nguyen\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000003SYxZIAW\\\",\\\"slackUserId\\\":\\\"WAS176048\\\",\\\"createdBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"modifiedBy\\\":\\\"tuannguyen@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:16.534Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:16.534Z\\\"},\\\"externalGusId\\\":null}],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Other\\\",\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73102086\\\",\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/ZHQDAc9AH2Vv\\\",\\\"engagedAt\\\":\\\"2025-03-17T06:29:55.775Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T07:07:22.220Z\\\"},\\\"eventObj\\\":{\\\"internalComms\\\":{\\\"sequenceNumber\\\":3,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Resolved\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"2025-03-17T07:03:00.000Z\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\\\",\\\"sentAt\\\":\\\"2025-03-17T07:08:18.220Z\\\",\\\"swimlaneUpdates\\\":[],\\\"slackChannelIds\\\":[\\\"C05U9CBHX8D\\\",\\\"C05T2A3C319\\\"],\\\"emailAddresses\\\":[\\\"gsm@salesforce.com\\\",\\\"sitereliability@salesforce.com\\\",\\\"techcustomerescalation@salesforce.com\\\"]},\\\"internalCommsId\\\":\\\"7ec916e0-1c07-43cb-9465-332d8575e78b\\\",\\\"allInternalCommsReport\\\":[{\\\"sequenceNumber\\\":3,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Resolved\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"2025-03-17T07:03:00.000Z\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\\\",\\\"sentAt\\\":\\\"2025-03-17T07:08:18.220Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":2,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Cen
2025-03-17T07:08:19.276537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] tric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\\\",\\\"sentAt\\\":\\\"2025-03-17T06:58:41.877Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":1,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Anna Marie Kaibigan\\\",\\\"incidentCommanderSlackUserId\\\":\\\"WBFEYNGA1\\\",\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\\\",\\\"actions\\\":\\\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\\\"},\\\"sentAt\\\":\\\"2025-03-17T06:45:09.128Z\\\",\\\"swimlaneUpdates\\\":[]}]},\\\"requestId\\\":\\\"dae6dabe-464a-b80b-a7b7-2e47459983e6\\\",\\\"client\\\":\\\"icc-slack-app\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T07:08:18.598Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"7ec916e0-1c07-43cb-9465-332d8575e78b\",\"allInternalCommsReport\":[{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud
2025-03-17T07:08:19.276537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] \":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"dae6dabe-464a-b80b-a7b7-2e47459983e6\",\"client\":\"icc-slack-app\"},\"encoder\":\"json\",\"eventType\":\"icc-internal-comms-request\",\"payload\":{\"createdAt\":\"2025-03-17T07:08:18.598Z\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"0ecd9085-134c-40b3-8327-b62637ef6fe0\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"externalGusId\":null},{\"createdAt\":\"2025-03-17T06:43:37.842Z\",\"modifiedAt\":\"2025-03-17T06:43:37.842Z\",\"createdBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"modifiedBy\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"id\":\"04d73ed0-dea3-4192-b83f-366a6de240a2\",\"name\":\"IncidentCommunicationsOfficer\",\"displayName\":\"Incident Communications Officer\",\"user\":{\"id\":\"7891c9ec-d110-4bca-9aa1-5dc1d6ede2c8\",\"email\":\"tuannguyen@salesforce.com\",\"active\":true,\"name\":\"Tuan Nguyen\",\"firstName\":\"Tuan\",\"lastName\":\"Nguyen\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000003SYxZIAW\",\"slackUserId\":\"WAS176048\",\"createdBy\":\"tuannguyen@salesforce.com\",\"modifiedBy\":\"tuannguyen@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:16.534Z\",\"modifiedAt\":\"2022-07-15T09:24:16.534Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"7ec916e0-1c07-43cb-9465-332d8575e78b\",\"allInternalCommsReport\":[{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"Resolved\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob is now running and all jobs have cleared and there is no more search delay as of 07:03 UTC as confirmed by the search team. This incident is now resolved.\",\"sentAt\":\"2025-03-17T07:08:18.220Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.
2025-03-17T07:08:19.276537+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] 000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":\"The cronjob was found to have been not working. We used a change case to restart the coreapp in which the cronjob was hosted to get us out of impact. The cronjob is now moving but we are currently waiting for the full clearing of the delays in the queue. Currently at a monitoring state as the jobs dequeues.\",\"sentAt\":\"2025-03-17T06:58:41.877Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/chy-nhbk-ufg\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Anna Marie Kaibigan\",\"incidentCommanderSlackUserId\":\"WBFEYNGA1\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"There is a search index delay on swe96s for over 24 hours which breached the SLA.\",\"actions\":\"We have paged out to the search team to triage the issue. The cron job isn't working and they'll need to execute a change case to fix the issue.\"},\"sentAt\":\"2025-03-17T06:45:09.128Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"dae6dabe-464a-b80b-a7b7-2e47459983e6\",\"client\":\"icc-slack-app\"}}"}
2025-03-17T07:08:19.276573+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299276,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T07:08:19.276603+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299276,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-internal-comms-request and docId bJBAAAadCyL"}
2025-03-17T07:08:19.276627+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195299276,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleInternalCommsRequest for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T07:09:07.473447+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195347473,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892378112,\"uptime\":2015242.86,\"totalMem\":4018151424,\"loadavg\":[0.01,0.05,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24196580,\"nice\":16990,\"sys\":3473130,\"idle\":1980106290,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23915910,\"nice\":16540,\"sys\":3489390,\"idle\":1980330660,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29824,\"num\":25,\"sum\":60015}}}"}
2025-03-17T07:09:07.473544+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195347473,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:13:07.525439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195587525,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891915264,\"uptime\":2015482.91,\"totalMem\":4018151424,\"loadavg\":[0,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24198410,\"nice\":16990,\"sys\":3473450,\"idle\":1980343300,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23917600,\"nice\":16540,\"sys\":3489720,\"idle\":1980567560,\"irq\":0}}],\"eventLoopStats\":{\"min\":2,\"max\":29826,\"num\":7,\"sum\":59999}}}"}
2025-03-17T07:13:07.525502+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195587525,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:16:07.581463+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195767581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890555392,\"uptime\":2015662.97,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24200120,\"nice\":16990,\"sys\":3473710,\"idle\":1980520950,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23918970,\"nice\":16540,\"sys\":3489920,\"idle\":1980745460,\"irq\":0}}],\"eventLoopStats\":{\"min\":189,\"max\":30030,\"num\":3,\"sum\":60025}}}"}
2025-03-17T07:16:07.581525+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195767581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:17:07.613438+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195827613,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890686464,\"uptime\":2015722.99,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24200720,\"nice\":16990,\"sys\":3473780,\"idle\":1980580190,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23919540,\"nice\":16540,\"sys\":3489980,\"idle\":1980804720,\"irq\":0}}],\"eventLoopStats\":{\"min\":193,\"max\":30013,\"num\":3,\"sum\":60028}}}"}
2025-03-17T07:17:07.613496+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195827613,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:17:17.682513+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:17:17.682583+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='bullmq-job-completed' for consumer group='icc-reporting'"}
2025-03-17T07:17:17.682585+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:17:17.631Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"},\"eventObj\":{\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"reminderId\":\"b423e979-fff0-4d5a-ad24-7ee94ef7e90c\",\"message\":\"Ensure Trust is Resolved\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"}},\"client\":\"icc-rest\"}"}
2025-03-17T07:17:17.682588+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T07:17:17.682670+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T07:17:17.682787+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:17:17.682787+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - bullmq-job-completed"}
2025-03-17T07:17:17.682788+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type bullmq-job-completed is not consumed by icc-reporting."}
2025-03-17T07:17:17.682805+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837682,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - bullmq-job-completed correctly routed"}
2025-03-17T07:17:17.770910+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:17:17.770913+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-reminder-triggered' for consumer group='icc-reporting'"}
2025-03-17T07:17:17.770916+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:17:17.716Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\"},\"eventObj\":{\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"reminderId\":\"b423e979-fff0-4d5a-ad24-7ee94ef7e90c\",\"message\":\"Ensure Trust is Resolved\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"reminderData\":{\"id\":\"b423e979-fff0-4d5a-ad24-7ee94ef7e90c\",\"type\":\"REMINDER\",\"active\":true,\"message\":\"Ensure Trust is Resolved\",\"audience\":[],\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"offsetUnit\":\"m\",\"triggersAt\":\"2025-03-17T07:17:17.563Z\",\"isRecurring\":false,\"occurrences\":[],\"offsetValue\":10,\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"currentOccurrence\":0,\"inheritedFromTeamConfig\":true},\"slackChannelId\":\"C08HYDW3893\"}}"}
2025-03-17T07:17:17.770917+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T07:17:17.770918+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T07:17:17.770934+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:17:17.773967+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-reminder-triggered"}
2025-03-17T07:17:17.773968+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-reminder-triggered is not consumed by icc-reporting."}
2025-03-17T07:17:17.773969+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195837770,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-reminder-triggered correctly routed"}
2025-03-17T07:18:07.621455+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742195887621,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888769536,\"uptime\":2015783,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24201430,\"nice\":16990,\"sys\":3473930,\"idle\":1980639070,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23920140,\"nice\":16540,\"sys\":3490110,\"idle\":1980863600,\"irq\":0}}],\"eventLoopStats\":{\"min\":2,\"max\":29812,\"num\":7,\"sum\":60005}}}"}
2025-03-17T07:18:07.621508+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742195887621,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:21:07.657412+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196067657,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887606272,\"uptime\":2015963.04,\"totalMem\":4018151424,\"loadavg\":[0.08,0.02,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24202700,\"nice\":16990,\"sys\":3474170,\"idle\":1980816700,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23921430,\"nice\":16540,\"sys\":3490390,\"idle\":1981041400,\"irq\":0}}],\"eventLoopStats\":{\"min\":185,\"max\":30019,\"num\":3,\"sum\":60006}}}"}
2025-03-17T07:21:07.657481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196067657,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:23:07.713374+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196187713,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887368704,\"uptime\":2016083.1,\"totalMem\":4018151424,\"loadavg\":[0.03,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24203980,\"nice\":16990,\"sys\":3474330,\"idle\":1980935120,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23922370,\"nice\":16540,\"sys\":3490560,\"idle\":1981160020,\"irq\":0}}],\"eventLoopStats\":{\"min\":196,\"max\":30011,\"num\":3,\"sum\":60028}}}"}
2025-03-17T07:23:07.713512+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196187713,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:25:28.608781+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328608,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:25:28.608952+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328608,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-to-cis-request' for consumer group='icc-reporting'"}
2025-03-17T07:25:28.608993+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328608,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:25:28.597Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-17T06:13:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"eventObj\":[{\"configurationItemId\":\"a7JEE00000J7QcN2AV\",\"lock\":false,\"gusCaseRecordId\":\"500EE00001TybJdYAJ\"}],\"requestId\":\"c0ca8ed9-fd9e-48ed-7019-1b0a435b6fdf\",\"client\":\"icc-slack-app\"}"}
2025-03-17T07:25:28.608997+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328608,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-request\"}}"}
2025-03-17T07:25:28.609056+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-request\"}}"}
2025-03-17T07:25:28.609680+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:25:28.609681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-to-cis-request"}
2025-03-17T07:25:28.609681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-incident-to-cis-request is not consumed by icc-reporting."}
2025-03-17T07:25:28.609681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-to-cis-request correctly routed"}
2025-03-17T07:25:28.609682+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:25:28.609683+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-resolved' for consumer group='icc-reporting'"}
2025-03-17T07:25:28.609685+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:25:28.350Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"changes\":{\"detectedTime\":[\"2025-03-17T06:13:00.000Z\",\"2025-03-16T20:27:00.000Z\"]},\"requestId\":\"c0ca8ed9-fd9e-48ed-7019-1b0a435b6fdf\",\"client\":\"icc-slack-app\"}"}
2025-03-17T07:25:28.609686+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-resolved\"}}"}
2025-03-17T07:25:28.609687+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-resolved\"}}"}
2025-03-17T07:25:28.609688+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:25:28.609688+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-resolved"}
2025-03-17T07:25:28.609688+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-incident-resolved is now being processed."}
2025-03-17T07:25:28.609688+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T07:25:28.610147+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,99,105,100,101,110,116,45,114,101,115,111,108,118,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,55,58,50,53,58,50,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T07:25:28.350Z\\\",\\\"user\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"icc-73102086\\\"},\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"description\\\":\\\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\\\nDetection Time: 2025-03-17 06:13:20.000 \\\\nReason: 50% Orgs Delayed for 24h \\\\n\\\\n\\\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\\\",\\\"detectedTime\\\":\\\"2025-03-16T20:27:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Resolved\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T07:03:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\\\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Other\\\",\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73102086\\\",\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/ZHQDAc9AH2Vv\\\",\\\"engagedAt\\\":\\\"2025-03-17T06:29:55.775Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T07:07:22.220Z\\\"},\\\"changes\\\":{\\\"detectedTime\\\":[\\\"2025-03-17T06:13:00.000Z\\\",\\\"2025-03-16T20:27:00.000Z\\\"]},\\\"requestId\\\":\\\"c0ca8ed9-fd9e-48ed-7019-1b0a435b6fdf\\\",\\\"client\\\":\\\"icc-slack-app\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T07:25:28.350Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"changes\":{\"detectedTime\":[\"2025-03-17T06:13:00.000Z\",\"2025-03-16T20:27:00.000Z\"]},\"requestId\":\"c0ca8ed9-fd9e-48ed-7019-1b0a435b6fdf\",\"client\":\"icc-slack-app\"},\"encoder\":\"json\",\"eventType\":\"icc-incident-resolved\",\"payload\":{\"createdAt\":\"2025-03-17T07:25:28.350Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"Sherlock detected Search Indexing Delay for  Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1 \\nDetection Time: 2025-03-17 06:13:20.000 \\nReason: 50% Orgs Delayed for 24h \\n\\n\\nCase created using .case by @Anna Marie Kaibigan in #cp-sre-alerts for swe96s\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:07:22.220Z\"},\"changes\":{\"detectedTime\":[\"2025-03-17T06:13:00.000Z\",\"2025-03-16T20:27:00.000Z\"]},\"requestId\":\"c0ca8ed9-fd9e-48ed-7019-1b0a435b6fdf\",\"client\":\"icc-slack-app\"}}"}
2025-03-17T07:25:28.610148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T07:25:28.610148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-incident-resolved and docId bJBAAAadCyL"}
2025-03-17T07:25:28.610148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Processing eventTypeIncidentResolved for 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:25:28.610149+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328610,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleIncidentResolved for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T07:25:28.610150+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328610,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: getIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info"}
2025-03-17T07:25:28.610357+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328610,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-gus-casenumber - value: [object Promise]"}
2025-03-17T07:25:28.654093+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328653,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:25:28.654159+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-report-generic-data' for consumer group='icc-reporting'"}
2025-03-17T07:25:28.654163+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"timestamp\":\"2025-03-17T07:25:28.598Z\",\"sectionTitle\":\"Incident to CI Updated \",\"contentObject\":{\"emptyData\":\"All Incident to CI have been removed\",\"dataTitle\":\"Incident to Ci updated with the following information:\",\"data\":[]}}"}
2025-03-17T07:25:28.654163+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-generic-data\"}}"}
2025-03-17T07:25:28.654231+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-generic-data\"}}"}
2025-03-17T07:25:28.654326+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:25:28.654506+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742196328654,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T07:25:33.077717+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:25:33.077783+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T07:25:33.077785+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:25:33.026Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T07:25:33.026Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}}"}
2025-03-17T07:25:33.077813+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T07:25:33.077871+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T07:25:33.078246+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:25:33.078247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-gus-response"}
2025-03-17T07:25:33.078248+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196333077,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T07:25:33.078253+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T07:25:33.078726+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,55,58,50,53,58,51,51,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T07:25:33.026Z\\\",\\\"user\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_resolved\\\",\\\"timestamp\\\":\\\"2025-03-17T07:25:33.026Z\\\",\\\"recordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"incidentId\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"caseNumber\\\":\\\"73102086\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T07:25:33.026Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T07:25:33.026Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T07:25:33.026Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TybJdYAJ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T07:25:33.026Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"},\"changes\":{}}}"}
2025-03-17T07:25:33.078729+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T07:25:33.078730+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-gus-response and docId bJBAAAadCyL"}
2025-03-17T07:25:33.078730+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleGusExport for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T07:25:33.078731+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6, docId bJBAAAadCyL, event {\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T07:25:33.026Z\",\"recordId\":\"500EE00001TybJdYAJ\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"73102086\"}"}
2025-03-17T07:25:33.078734+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-gus-casenumber - value: [object Promise]"}
2025-03-17T07:25:33.078852+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6, docId bJBAAAadCyL, caseNumber 73102086"}
2025-03-17T07:25:33.078852+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196333078,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-gus-response correctly routed"}
2025-03-17T07:28:07.781419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196487781,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886909952,\"uptime\":2016383.17,\"totalMem\":4018151424,\"loadavg\":[0.04,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24205710,\"nice\":16990,\"sys\":3474730,\"idle\":1981231320,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23924560,\"nice\":16540,\"sys\":3490920,\"idle\":1981456260,\"irq\":0}}],\"eventLoopStats\":{\"min\":155,\"max\":30017,\"num\":3,\"sum\":60005}}}"}
2025-03-17T07:28:07.781478+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196487781,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:29:07.801445+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196547801,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886909952,\"uptime\":2016443.18,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24206420,\"nice\":16990,\"sys\":3474810,\"idle\":1981290500,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23925100,\"nice\":16540,\"sys\":3491000,\"idle\":1981515570,\"irq\":0}}],\"eventLoopStats\":{\"min\":166,\"max\":30024,\"num\":3,\"sum\":60017}}}"}
2025-03-17T07:29:07.801532+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196547801,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:07.058867+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607058,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:30:07.058936+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196607058,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-bridge-folder-created' for consumer group='icc-reporting'"}
2025-03-17T07:30:07.058937+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607058,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:30:06.879Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"18wMUKIgBxFnlFGvGSF-alC7bxi_DeMoh\"}}"}
2025-03-17T07:30:07.058938+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607058,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-folder-created\"}}"}
2025-03-17T07:30:07.059027+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607058,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-folder-created\"}}"}
2025-03-17T07:30:07.059120+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:07.059120+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-bridge-folder-created"}
2025-03-17T07:30:07.059121+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-bridge-folder-created is now being processed."}
2025-03-17T07:30:07.059294+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T07:30:07.061912+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId bJBAAAadCyL and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,98,114,105,100,103,101,45,102,111,108,100,101,114,45,99,114,101,97,116,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,55,58,51,48,58,48,54,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,98,114,105,100,103,101]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T07:30:06.879Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"NA\\\",\\\"name\\\":\\\"NA\\\"}},\\\"eventObj\\\":{\\\"folderId\\\":\\\"18wMUKIgBxFnlFGvGSF-alC7bxi_DeMoh\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T07:30:06.879Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"18wMUKIgBxFnlFGvGSF-alC7bxi_DeMoh\"}},\"encoder\":\"json\",\"eventType\":\"icc-bridge-folder-created\",\"payload\":{\"createdAt\":\"2025-03-17T07:30:06.879Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"18wMUKIgBxFnlFGvGSF-alC7bxi_DeMoh\"}}}"}
2025-03-17T07:30:07.061915+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: bJBAAAadCyL"}
2025-03-17T07:30:07.061916+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-bridge-folder-created and docId bJBAAAadCyL"}
2025-03-17T07:30:07.061919+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.bridgeRecordingFolderCreated for id 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and docId bJBAAAadCyL"}
2025-03-17T07:30:07.724226+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196607724,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-bridge-folder-created correctly routed"}
2025-03-17T07:30:07.801417+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196607801,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886975488,\"uptime\":2016503.18,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24207160,\"nice\":16990,\"sys\":3474890,\"idle\":1981349640,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23925630,\"nice\":16540,\"sys\":3491080,\"idle\":1981574830,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30030,\"num\":24,\"sum\":59989}}}"}
2025-03-17T07:30:07.801504+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196607801,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:09.621597+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196609621,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6-incident-info, info: {\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseId\":\"500EE00001TybJdYAJ\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"slackChannelId\":\"C08HYDW3893\",\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\n\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T04:14:00.000Z\",\"impactEnd\":\"2025-03-17T07:03:00.000Z\",\"servicesImpacted\":\"Core Live on Falcon\",\"roles\":{},\"category\":\"Application\",\"rootCause\":\"Other\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"caseNumber\":\"73102086\",\"productTag\":\"a1aEE0000003rSnYAI\",\"proximateCause\":\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\",\"customerExperience\":\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\",\"immediateResolution\":\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\",\"aarQuestions\":\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Anna Marie Kaibigan\"}"}
2025-03-17T07:30:09.622999+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196609622,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages Adding Slack messages to docId bJBAAAadCyL..."}
2025-03-17T07:30:10.213371+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610213,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02Q288NF7W"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.215199+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610215,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U01UPBJAV5F"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.220485+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610220,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02GXBX8PUZ"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.241252+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610241,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02DWLVTKB9"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.245747+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U032QNNKMT7"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.250644+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610250,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U03LNRS637U"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:10.254179+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196610254,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U06Q54CS88M"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T07:30:12.049289+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612049,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages remove redis keys for docId bJBAAAadCyL"}
2025-03-17T07:30:12.051101+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612050,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages produceEvent for docId bJBAAAadCyL"}
2025-03-17T07:30:12.051103+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"produceEvent with link https://salesforce.quip.com/ZHQDAc9AH2Vv for event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 and case 73102086"}
2025-03-17T07:30:12.051108+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricRequestExecute] - icc-report-finished - undefined"}
2025-03-17T07:30:12.051148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.051177+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.051283+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:12.051463+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key '4a6faf32-c583-4c0e-b53f-ee1b00fd55c6' to topic 'incident': {\"createdAt\":\"2025-03-17T07:30:12.051Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\n\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"closingDetails\":{\"productTag\":\"a1aEE0000003rSnYAI\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\",\"customerExperience\":\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\",\"immediateResolution\":\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\",\"aarQuestions\":\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SearchOps\",\"closedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"createdBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"modifiedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T07:30:06.434Z\",\"modifiedAt\":\"2025-03-17T07:30:06.434Z\",\"id\":\"c1839112-8f6e-4541-8eaf-679591adaf75\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:25:33.026Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\"}}"}
2025-03-17T07:30:12.051529+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key '4a6faf32-c583-4c0e-b53f-ee1b00fd55c6' to Kafka topic 'incident'"}
2025-03-17T07:30:12.051530+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricDurationTime] - icc-report-finished - undefined"}
2025-03-17T07:30:12.051550+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.23295199871063232\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.051606+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.23295199871063232\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.051624+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:12.051651+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612051,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-incident-closed correctly routed"}
2025-03-17T07:30:12.060047+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612059,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:30:12.060443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-report-finished' for consumer group='icc-reporting'"}
2025-03-17T07:30:12.060538+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:30:12.051Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\n\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"closingDetails\":{\"productTag\":\"a1aEE0000003rSnYAI\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\",\"customerExperience\":\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\",\"immediateResolution\":\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\",\"aarQuestions\":\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SearchOps\",\"closedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"createdBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"modifiedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T07:30:06.434Z\",\"modifiedAt\":\"2025-03-17T07:30:06.434Z\",\"id\":\"c1839112-8f6e-4541-8eaf-679591adaf75\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:25:33.026Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\"}}"}
2025-03-17T07:30:12.060539+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.060584+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T07:30:12.060660+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:12.060690+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-report-finished"}
2025-03-17T07:30:12.060691+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-report-finished is now being processed."}
2025-03-17T07:30:12.060807+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612060,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - value: [object Promise]"}
2025-03-17T07:30:12.061930+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612061,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,45,102,105,110,105,115,104,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,55,58,51,48,58,49,50,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,105,110,103]}}],\"key\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T07:30:12.051Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08HYDW3893\\\",\\\"name\\\":\\\"icc-73102086\\\"},\\\"subject\\\":\\\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\\\",\\\"description\\\":\\\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\\\n\\\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\\\",\\\"detectedTime\\\":\\\"2025-03-16T20:27:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Closed\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T04:14:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T07:03:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/chy-nhbk-ufg\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\\\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Other\\\",\\\"createdAt\\\":\\\"2025-03-17T06:29:41.802Z\\\",\\\"closingDetails\\\":{\\\"productTag\\\":\\\"a1aEE0000003rSnYAI\\\",\\\"detectedByServiceOwner\\\":true,\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"proximateCause\\\":\\\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\\\",\\\"customerExperience\\\":\\\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\\\",\\\"immediateResolution\\\":\\\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\\\",\\\"aarQuestions\\\":\\\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\\\",\\\"changes\\\":\\\"\\\",\\\"breakFix\\\":\\\"\\\",\\\"productTagName\\\":\\\"SearchOps\\\",\\\"closedBy\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"firstName\\\":\\\"Anna\\\",\\\"lastName\\\":\\\"Marie\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"createdBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"modifiedBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-05T23:58:47.585Z\\\",\\\"modifiedAt\\\":\\\"2022-04-05T23:58:47.585Z\\\"},\\\"incidentId\\\":\\\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"firstName\\\":\\\"Anna\\\",\\\"lastName\\\":\\\"Marie\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"createdBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"modifiedBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-05T23:58:47.585Z\\\",\\\"modifiedAt\\\":\\\"2022-04-05T23:58:47.585Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\\\",\\\"email\\\":\\\"akaibigan@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Anna Marie Kaibigan\\\",\\\"firstName\\\":\\\"Anna\\\",\\\"lastName\\\":\\\"Marie\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000000SS1mIAG\\\",\\\"slackUserId\\\":\\\"WBFEYNGA1\\\",\\\"createdBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"modifiedBy\\\":\\\"akaibigan@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-05T23:58:47.585Z\\\",\\\"modifiedAt\\\":\\\"2022-04-05T23:58:47.585Z\\\"},\\\"closingComments\\\":null,\\\"createdAt\\\":\\\"2025-03-17T07:30:06.434Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T07:30:06.434Z\\\",\\\"id\\\":\\\"c1839112-8f6e-4541-8eaf-679591adaf75\\\"},\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73102086\\\",\\\"caseRecordId\\\":\\\"500EE00001TybJdYAJ\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/ZHQDAc9AH2Vv\\\",\\\"engagedAt\\\":\\\"2025-03-17T06:29:55.775Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T07:25:33.026Z\\\"},\\\"eventObj\\\":{\\\"link\\\":\\\"https://salesforce.quip.com/ZHQDAc9AH2Vv\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T07:30:12.051Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\n\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"closingDetails\":{\"productTag\":\"a1aEE0000003rSnYAI\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction
2025-03-17T07:30:12.061930+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\",\"customerExperience\":\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\",\"immediateResolution\":\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\",\"aarQuestions\":\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SearchOps\",\"closedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"createdBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"modifiedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T07:30:06.434Z\",\"modifiedAt\":\"2025-03-17T07:30:06.434Z\",\"id\":\"c1839112-8f6e-4541-8eaf-679591adaf75\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:25:33.026Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\"}},\"encoder\":\"json\",\"eventType\":\"icc-report-finished\",\"payload\":{\"createdAt\":\"2025-03-17T07:30:12.051Z\",\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"icc-73102086\"},\"subject\":\"(Warden AIOps) Search Indexing Delay for Falcon Cell: swe96s, FI: aws-prod16-eunorth1 Rac Node: 1\",\"description\":\"On March 17, 2025, a search index delay incident was detected on the Falcon Cell instance 'swe96s', affecting the search functionality and breaching the Service Level Agreement (SLA). The delay impacted 50% of organizations, causing performance degradation for over 24 hours. The incident was first detected by Sherlock and was addressed by the on-call search team. The issue was traced to a non-functional cron job responsible for managing the search index. The cron was not running since March 16 04 UTC. Search team have created a change case and submitted an app restart through Fawkes self-service however the approval did not went in. Since cron was not running, the requests were not being processed and breached trust posting SLA of 24 hours. Immediate intervention was required to restore the search functionality and mitigate customer impact. The incident was resolved within 169 minutes after restarting the core application hosting the cron job, which resumed normal operations.\\n\\nhttps://salesforce-internal.slack.com/archives/C028H4A90H2/p1742156841327659\\nhttps://gus.lightning.force.com/lightning/r/Case/500EE00001TxgObYAJ/view\\nhttps://salesforce-internal.slack.com/archives/C045V74G7RB/p1742155039628049\",\"detectedTime\":\"2025-03-16T20:27:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q0YAZRT246FLY2\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T04:14:00.000Z\",\"impactEndTime\":\"2025-03-17T07:03:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/chy-nhbk-ufg\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-385-404-0825, pin: 112204856; \\nFind numbers in more regions on https://tel.meet/chy-nhbk-ufg?pin=5057668449474, pin: 5057668449474\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Other\",\"createdAt\":\"2025-03-17T06:29:41.802Z\",\"closingDetails\":{\"productTag\":\"a1aEE0000003rSnYAI\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The main cause of the incident was identified as a non-functional cron job responsible for managing the search index on the instance 'swe96s'. This malfunction led to a significant delay in search indexing, affecting half of the organizations for a day. The issue required restarting the core application hosting the cron job to restore its functionality.\",\"customerExperience\":\"There’s a Salesforce Search performance degradation where some users may experience difficulties creating or updating object records. Additionally, some users may be unable to view recently created records.\",\"immediateResolution\":\"The immediate action taken was to restart the core application hosting the non-functional cron job responsible for the search index. This action was part of a change case aimed at mitigating the impact. The cron job resumed operations, and the search queue began to clear, resolving the issue.\",\"aarQuestions\":\"What preventive measures can be implemented to ensure cron jobs remain functional and do not fail unexpectedly?\\nHow can monitoring and alerting systems be improved to detect similar issues earlier and reduce resolution time?\\nWhat training or documentation updates are needed to ensure on-call engineers can quickly identify and resolve similar issues in the future?\\nReview of Team Dependency between CAPE and Search team with regards to Fawkes Self-Service process.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SearchOps\",\"closedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"createdBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"modifiedBy\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"email\":\"akaibigan@salesforce.com\",\"active\":true,\"name\":\"Anna Marie Kaibigan\",\"firstName\":\"Anna\",\"lastName\":\"Marie\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"createdBy\":\"akaibigan@salesforce.com\",\"modifiedBy\":\"akaibigan@salesforce.com\",\"createdAt\":\"2022-04-05T23:58:47.585Z\",\"modifiedAt\":\"2022-04-05T23:58:47.585Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T07:30:06.434Z\",\"modifiedAt\":\"2025-03-17T07:30:06.434Z\",\"id\":\"c1839112-8f6e-4541-8eaf-679591adaf75\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73102086\",\"caseRecordId\":\"500EE00001TybJdYAJ\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\",\"engagedAt\":\"2025-03-17T06:29:55.775Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T07:25:33.026Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/ZHQDAc9AH2Vv\"}}}"}
2025-03-17T07:30:12.061931+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196612061,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:30:12.062085+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742196612062,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-report-finished does not have the necessary information to create a document"}
2025-03-17T07:30:12.062086+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196612062,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-report-finished correctly routed"}
2025-03-17T07:30:18.194442+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196618193,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6"}
2025-03-17T07:30:18.194447+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196618193,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-prb-number-updated' for consumer group='icc-reporting'"}
2025-03-17T07:30:18.194449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196618193,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T07:30:17.519Z\",\"user\":{\"id\":\"079d8dda-009e-46b8-a63f-ba1b51dfcb15\",\"gusUserId\":\"005B0000000SS1mIAG\",\"slackUserId\":\"WBFEYNGA1\",\"name\":\"Anna Marie Kaibigan\",\"email\":\"akaibigan@salesforce.com\"},\"incident\":{\"id\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"slackChannel\":{\"id\":\"C08HYDW3893\",\"name\":\"\"},\"caseRecordId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"prbNumber\":{\"sfid\":\"a4QEE00000067zh2AA\",\"valid\":true,\"gusIdentifier\":\"PRB-0027120\",\"value\":\"PRB-0027120\",\"createdByIcc\":true,\"workItems\":[{\"id\":\"a07EE00002BA4a5YAD\",\"success\":true}]}},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T07:30:17.519Z\",\"recordId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"incidentId\":\"4a6faf32-c583-4c0e-b53f-ee1b00fd55c6\",\"caseNumber\":\"\"}}"}
2025-03-17T07:30:18.194450+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-prb-number-updated\"}}"}
2025-03-17T07:30:18.194467+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-prb-number-updated\"}}"}
2025-03-17T07:30:18.194468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:30:18.194469+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-prb-number-updated"}
2025-03-17T07:30:18.194469+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-prb-number-updated is not consumed by icc-reporting."}
2025-03-17T07:30:18.194470+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196618194,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 4a6faf32-c583-4c0e-b53f-ee1b00fd55c6 - icc-prb-number-updated correctly routed"}
2025-03-17T07:32:07.829386+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742196727829,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886823936,\"uptime\":2016623.21,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24208200,\"nice\":16990,\"sys\":3475090,\"idle\":1981467890,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23926510,\"nice\":16540,\"sys\":3491310,\"idle\":1981693350,\"irq\":0}}],\"eventLoopStats\":{\"min\":18,\"max\":30030,\"num\":4,\"sum\":60025}}}"}
2025-03-17T07:32:07.829517+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196727829,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:35:13.671154+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196913670,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T07:35:13.671828+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742196913671,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T07:51:08.261449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742197868261,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894774272,\"uptime\":2017763.64,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24218940,\"nice\":16990,\"sys\":3476650,\"idle\":1982593330,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23936620,\"nice\":16540,\"sys\":3492920,\"idle\":1982818780,\"irq\":0}}],\"eventLoopStats\":{\"min\":120,\"max\":30005,\"num\":3,\"sum\":60008}}}"}
2025-03-17T07:51:08.261590+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742197868261,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T07:51:24.233848+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742197884233,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T07:51:24.234719+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742197884234,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T08:03:08.497641+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742198588497,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893111296,\"uptime\":2018483.88,\"totalMem\":4018151424,\"loadavg\":[0.16,0.05,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24225170,\"nice\":16990,\"sys\":3477620,\"idle\":1983304230,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23941540,\"nice\":16540,\"sys\":3493970,\"idle\":1983529920,\"irq\":0}}],\"eventLoopStats\":{\"min\":135,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T08:03:08.497703+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742198588497,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:06:08.581442+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742198768581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2894458880,\"uptime\":2018663.96,\"totalMem\":4018151424,\"loadavg\":[0,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24226200,\"nice\":16990,\"sys\":3477870,\"idle\":1983481850,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23942640,\"nice\":16540,\"sys\":3494240,\"idle\":1983707750,\"irq\":0}}],\"eventLoopStats\":{\"min\":88,\"max\":30027,\"num\":3,\"sum\":60023}}}"}
2025-03-17T08:06:08.581489+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742198768581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:06:27.544453+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742198787544,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T08:06:27.545193+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742198787545,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T08:09:08.645407+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742198948645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2895331328,\"uptime\":2018844.03,\"totalMem\":4018151424,\"loadavg\":[0.02,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24228770,\"nice\":16990,\"sys\":3478110,\"idle\":1983659490,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23944360,\"nice\":16540,\"sys\":3494420,\"idle\":1983885650,\"irq\":0}}],\"eventLoopStats\":{\"min\":53,\"max\":30018,\"num\":3,\"sum\":60015}}}"}
2025-03-17T08:09:08.645468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742198948645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:10:08.657386+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199008657,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893901824,\"uptime\":2018904.04,\"totalMem\":4018151424,\"loadavg\":[0.06,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24229510,\"nice\":16990,\"sys\":3478200,\"idle\":1983718650,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23944970,\"nice\":16540,\"sys\":3494500,\"idle\":1983944920,\"irq\":0}}],\"eventLoopStats\":{\"min\":56,\"max\":30008,\"num\":3,\"sum\":60012}}}"}
2025-03-17T08:10:08.657494+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199008657,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:12:08.693482+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199128693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893221888,\"uptime\":2019024.08,\"totalMem\":4018151424,\"loadavg\":[0.01,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24230910,\"nice\":16990,\"sys\":3478320,\"idle\":1983837190,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23945970,\"nice\":16540,\"sys\":3494650,\"idle\":1984063390,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29966,\"num\":13,\"sum\":60026}}}"}
2025-03-17T08:12:08.693494+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199128693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:12:48.891014+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199168890,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:12:48.891237+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-bridge-generated' for consumer group='icc-reporting'"}
2025-03-17T08:12:48.891284+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:12:48.881Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-8314687c-abb3-4165-bbbf-414b12112a6b\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"New\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\",\"isManaged\":true},\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"}}"}
2025-03-17T08:12:48.891287+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-generated\"}}"}
2025-03-17T08:12:48.891337+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-generated\"}}"}
2025-03-17T08:12:48.891432+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:12:48.891443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-bridge-generated"}
2025-03-17T08:12:48.891468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-bridge-generated is not consumed by icc-reporting."}
2025-03-17T08:12:48.891510+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199168891,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-bridge-generated correctly routed"}
2025-03-17T08:12:52.869841+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172869,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:12:52.870011+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199172869,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T08:12:52.870012+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172869,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:12:52.819Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_created\",\"timestamp\":\"2025-03-17T08:12:52.819Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{\"lastExportStatus\":[null,\"success\"]}}"}
2025-03-17T08:12:52.870015+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172869,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T08:12:52.870081+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T08:12:52.870192+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:12:52.870193+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response"}
2025-03-17T08:12:52.870194+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T08:12:52.870349+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T08:12:52.870666+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId ODAAAAeWWAB and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,56,58,49,50,58,53,50,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T08:12:52.819Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_created\\\",\\\"timestamp\\\":\\\"2025-03-17T08:12:52.819Z\\\",\\\"recordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"incidentId\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"caseNumber\\\":\\\"73104263\\\"},\\\"changes\\\":{\\\"lastExportStatus\\\":[null,\\\"success\\\"]}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T08:12:52.819Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_created\",\"timestamp\":\"2025-03-17T08:12:52.819Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{\"lastExportStatus\":[null,\"success\"]}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T08:12:52.819Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_created\",\"timestamp\":\"2025-03-17T08:12:52.819Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{\"lastExportStatus\":[null,\"success\"]}}}"}
2025-03-17T08:12:52.870681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: ODAAAAeWWAB"}
2025-03-17T08:12:52.870682+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-gus-response and docId ODAAAAeWWAB"}
2025-03-17T08:12:52.870683+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleGusExport for id 8314687c-abb3-4165-bbbf-414b12112a6b and docId ODAAAAeWWAB"}
2025-03-17T08:12:52.870687+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, event {\"status\":\"success\",\"event\":\"incident_created\",\"timestamp\":\"2025-03-17T08:12:52.819Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"}"}
2025-03-17T08:12:52.870762+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b-gus-casenumber - value: [object Promise]"}
2025-03-17T08:12:52.870918+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, caseNumber null"}
2025-03-17T08:12:52.870919+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199172870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.setValue key: 8314687c-abb3-4165-bbbf-414b12112a6b-gus-casenumber - value: 73104263"}
2025-03-17T08:12:58.947737+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178947,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:12:58.948058+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-updated' for consumer group='icc-reporting'"}
2025-03-17T08:12:58.948151+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:12:58.804Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:12:52.819Z\",\"usdResources\":[]},\"changes\":{\"roles\":{\"0\":[null,{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}]},\"servicesImpacted\":{\"0\":[null,\"Core Live on Falcon\"]}},\"requestId\":\"86a6025c-65bd-8e97-5237-516c75938e28\",\"client\":\"quantumks2s\"}"}
2025-03-17T08:12:58.948152+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-updated\"}}"}
2025-03-17T08:12:58.948229+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-updated\"}}"}
2025-03-17T08:12:58.948320+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:12:58.948320+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-incident-updated"}
2025-03-17T08:12:58.948321+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-incident-updated is now being processed."}
2025-03-17T08:12:58.948447+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T08:12:58.948944+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId ODAAAAeWWAB and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,99,105,100,101,110,116,45,117,112,100,97,116,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,56,58,49,50,58,53,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T08:12:58.804Z\\\",\\\"user\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"name\\\":\\\"SiteReliability\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"icc-73104263\\\"},\\\"subject\\\":\\\"APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"description\\\":\\\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\\\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\\\n\\\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\\\",\\\"detectedTime\\\":\\\"2025-03-17T08:06:02.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hfi-qyjv-nzz\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\\\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\\\n\\\"},\\\"roles\\\":[{\\\"createdAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"id\\\":\\\"9612e4ea-7f9f-4836-957a-9d07056c2237\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"externalGusId\\\":null}],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73104263\\\",\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/m6mBA2aDHeqm\\\",\\\"engagedAt\\\":\\\"2025-03-17T08:12:58.310Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T08:12:52.819Z\\\",\\\"usdResources\\\":[]},\\\"changes\\\":{\\\"roles\\\":{\\\"0\\\":[null,{\\\"createdAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"id\\\":\\\"9612e4ea-7f9f-4836-957a-9d07056c2237\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"externalGusId\\\":null}]},\\\"servicesImpacted\\\":{\\\"0\\\":[null,\\\"Core Live on Falcon\\\"]}},\\\"requestId\\\":\\\"86a6025c-65bd-8e97-5237-516c75938e28\\\",\\\"client\\\":\\\"quantumks2s\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T08:12:58.804Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a009
2025-03-17T08:12:58.948944+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] 0000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:12:52.819Z\",\"usdResources\":[]},\"changes\":{\"roles\":{\"0\":[null,{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}]},\"servicesImpacted\":{\"0\":[null,\"Core Live on Falcon\"]}},\"requestId\":\"86a6025c-65bd-8e97-5237-516c75938e28\",\"client\":\"quantumks2s\"},\"encoder\":\"json\",\"eventType\":\"icc-incident-updated\",\"payload\":{\"createdAt\":\"2025-03-17T08:12:58.804Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:12:52.819Z\",\"usdResources\":[]},\"changes\":{\"roles\":{\"0\":[null,{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}]},\"servicesImpacted\":{\"0\":[null,\"Core Live on Falcon\"]}},\"requestId\":\"86a6025c-65bd-8e97-5237-516c75938e28\",\"client\":\"quantumks2s\"}}"}
2025-03-17T08:12:58.948946+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: ODAAAAeWWAB"}
2025-03-17T08:12:58.948946+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-incident-updated and docId ODAAAAeWWAB"}
2025-03-17T08:12:58.948947+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleIncidentUpdated for id 8314687c-abb3-4165-bbbf-414b12112a6b and docId ODAAAAeWWAB"}
2025-03-17T08:12:58.948964+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178948,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: getIncidentInfo, key: 8314687c-abb3-4165-bbbf-414b12112a6b-incident-info"}
2025-03-17T08:12:58.949280+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199178949,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b-gus-casenumber - value: [object Promise]"}
2025-03-17T08:13:03.633639+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:13:03.633832+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T08:13:03.633834+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:13:03.583Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T08:13:03.583Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}"}
2025-03-17T08:13:03.633834+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T08:13:03.633885+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T08:13:03.633990+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:13:03.633991+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response"}
2025-03-17T08:13:03.634025+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199183633,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T08:13:03.634188+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T08:13:03.634533+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId ODAAAAeWWAB and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,56,58,49,51,58,48,51,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T08:13:03.583Z\\\",\\\"user\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"name\\\":\\\"SiteReliability\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_updated\\\",\\\"timestamp\\\":\\\"2025-03-17T08:13:03.583Z\\\",\\\"recordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"incidentId\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"caseNumber\\\":\\\"73104263\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T08:13:03.583Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T08:13:03.583Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T08:13:03.583Z\",\"user\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"name\":\"SiteReliability\",\"email\":\"sitereliability@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T08:13:03.583Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}}"}
2025-03-17T08:13:03.634534+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: ODAAAAeWWAB"}
2025-03-17T08:13:03.634535+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-gus-response and docId ODAAAAeWWAB"}
2025-03-17T08:13:03.634535+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleGusExport for id 8314687c-abb3-4165-bbbf-414b12112a6b and docId ODAAAAeWWAB"}
2025-03-17T08:13:03.634556+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, event {\"status\":\"success\",\"event\":\"incident_updated\",\"timestamp\":\"2025-03-17T08:13:03.583Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"}"}
2025-03-17T08:13:03.634642+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b-gus-casenumber - value: [object Promise]"}
2025-03-17T08:13:03.634806+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, caseNumber 73104263"}
2025-03-17T08:13:03.634807+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199183634,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response correctly routed"}
2025-03-17T08:14:57.267106+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199297266,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-request correctly routed"}
2025-03-17T08:14:57.348353+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:14:57.348627+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T08:14:57.348711+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:14:57.338Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:13:04.655Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"C05U9CBHX8D\",\"deliveryStatus\":true,\"type\":\"SlackChannelId\"},{\"notification\":\"C05T2A3C319\",\"deliveryStatus\":true,\"type\":\"SlackChannelId\"}],\"internalCommsId\":\"b5000745-a9d2-4292-9f36-bdba32184df4\",\"sequenceNumber\":1}}"}
2025-03-17T08:14:57.348712+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:14:57.348749+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:14:57.348881+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:14:57.348882+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response"}
2025-03-17T08:14:57.348883+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T08:14:57.348906+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199297348,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response correctly routed"}
2025-03-17T08:19:04.404623+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544404,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:19:04.404882+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544404,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T08:19:04.404921+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544404,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:19:04.391Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:13:04.655Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"2mfwus-5qc7j4-o0s1j2f-pehef-1742199544162\",\"client\":\"icc-ui\"}"}
2025-03-17T08:19:04.404959+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544404,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T08:19:04.404997+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544404,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T08:19:04.405141+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544405,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:19:04.405141+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544405,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted"}
2025-03-17T08:19:04.405162+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544405,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T08:19:04.405174+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544405,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted correctly routed"}
2025-03-17T08:19:04.422102+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:19:04.422344+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-created' for consumer group='icc-reporting'"}
2025-03-17T08:19:04.422391+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:19:04.413Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:02.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T08:12:44.236Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:13:04.655Z\"},\"eventObj\":[],\"requestId\":\"2mfwus-5qc7j4-o0s1j2f-pehef-1742199544162\",\"client\":\"icc-ui\"}"}
2025-03-17T08:19:04.422419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T08:19:04.422458+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T08:19:04.422551+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:19:04.422552+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-created"}
2025-03-17T08:19:04.422571+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-created is not consumed by icc-reporting."}
2025-03-17T08:19:04.422571+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544422,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-created correctly routed"}
2025-03-17T08:19:04.477252+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:19:04.477365+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-response' for consumer group='icc-reporting'"}
2025-03-17T08:19:04.477401+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:19:04.422Z\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"responses\":[]}"}
2025-03-17T08:19:04.477416+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T08:19:04.477475+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T08:19:04.477570+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:19:04.477796+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742199544477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T08:19:08.769410+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199548769,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888667136,\"uptime\":2019444.15,\"totalMem\":4018151424,\"loadavg\":[0.04,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24234000,\"nice\":16990,\"sys\":3478850,\"idle\":1984251130,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23949400,\"nice\":16540,\"sys\":3495220,\"idle\":1984477770,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":25582,\"num\":46,\"sum\":59979}}}"}
2025-03-17T08:19:08.769454+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199548769,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:20:08.769408+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199608769,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888667136,\"uptime\":2019504.15,\"totalMem\":4018151424,\"loadavg\":[0.01,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24234470,\"nice\":16990,\"sys\":3478920,\"idle\":1984310360,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23949950,\"nice\":16540,\"sys\":3495280,\"idle\":1984537010,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":27015,\"num\":16,\"sum\":59995}}}"}
2025-03-17T08:20:08.769450+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199608769,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:23:08.813436+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199788813,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887757824,\"uptime\":2019684.2,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24236280,\"nice\":16990,\"sys\":3479140,\"idle\":1984487940,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23951370,\"nice\":16540,\"sys\":3495500,\"idle\":1984714790,\"irq\":0}}],\"eventLoopStats\":{\"min\":2,\"max\":30027,\"num\":8,\"sum\":60006}}}"}
2025-03-17T08:23:08.813504+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199788813,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:24:08.829476+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742199848829,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2887757824,\"uptime\":2019744.21,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24237300,\"nice\":16990,\"sys\":3479200,\"idle\":1984547150,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23951690,\"nice\":16540,\"sys\":3495580,\"idle\":1984774100,\"irq\":0}}],\"eventLoopStats\":{\"min\":120,\"max\":30023,\"num\":3,\"sum\":60017}}}"}
2025-03-17T08:24:08.829536+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199848829,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:24:10.370929+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199850370,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T08:24:10.371676+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742199850371,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T08:26:52.696934+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200012696,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:26:52.697247+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-page-request' for consumer group='icc-reporting'"}
2025-03-17T08:26:52.697291+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:26:52.685Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"ok\":true,\"id\":\"db552bc4-61a4-42f3-871b-8e10993ec5b1\",\"serviceId\":\"P25VK5X\",\"serviceName\":\"GUS: CAPE Incident Escalation\",\"customMessage\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \",\"incidentSubject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"incidentDescription\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\"},\"requestId\":\"jxu8v-3qnfe-c2lej-ziuadg-1742200012298\",\"client\":\"icc-ui\"}"}
2025-03-17T08:26:52.697348+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T08:26:52.697421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-request\"}}"}
2025-03-17T08:26:52.697533+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:26:52.697554+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-page-request"}
2025-03-17T08:26:52.697558+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-page-request is not consumed by icc-reporting."}
2025-03-17T08:26:52.697571+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200012697,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-page-request correctly routed"}
2025-03-17T08:29:08.957412+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200148957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2893115392,\"uptime\":2020044.34,\"totalMem\":4018151424,\"loadavg\":[0,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24238990,\"nice\":16990,\"sys\":3479530,\"idle\":1984843520,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23953440,\"nice\":16540,\"sys\":3495960,\"idle\":1985070270,\"irq\":0}}],\"eventLoopStats\":{\"min\":119,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T08:29:08.957517+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200148957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:32:09.001453+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200329001,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2890821632,\"uptime\":2020224.39,\"totalMem\":4018151424,\"loadavg\":[0.04,0.03,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24240920,\"nice\":16990,\"sys\":3479800,\"idle\":1985021230,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23955440,\"nice\":16540,\"sys\":3496190,\"idle\":1985247970,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30027,\"num\":14,\"sum\":60010}}}"}
2025-03-17T08:32:09.001633+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200329001,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:34:09.033396+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200449033,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2891071488,\"uptime\":2020344.42,\"totalMem\":4018151424,\"loadavg\":[0.07,0.05,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24241600,\"nice\":16990,\"sys\":3479960,\"idle\":1985139850,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23956650,\"nice\":16540,\"sys\":3496360,\"idle\":1985366340,\"irq\":0}}],\"eventLoopStats\":{\"min\":134,\"max\":30028,\"num\":3,\"sum\":60027}}}"}
2025-03-17T08:34:09.033516+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200449033,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:40:26.023510+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:40:26.023773+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T08:40:26.023849+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:40:25.928Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"cf582ec5-f906-4e2a-9e8e-d0ef30ad0d80\",\"sequenceNumber\":2}}"}
2025-03-17T08:40:26.023852+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:40:26.023886+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:40:26.024008+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:40:26.024009+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response"}
2025-03-17T08:40:26.024012+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200826023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T08:40:26.024036+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200826024,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response correctly routed"}
2025-03-17T08:43:09.249419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742200989249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2892685312,\"uptime\":2020884.63,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24245590,\"nice\":16990,\"sys\":3480710,\"idle\":1985673140,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23960870,\"nice\":16540,\"sys\":3497020,\"idle\":1985899550,\"irq\":0}}],\"eventLoopStats\":{\"min\":90,\"max\":30026,\"num\":3,\"sum\":60029}}}"}
2025-03-17T08:43:09.249540+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742200989249,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:53:09.357403+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201589357,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2874130432,\"uptime\":2021484.74,\"totalMem\":4018151424,\"loadavg\":[0.01,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24250790,\"nice\":16990,\"sys\":3481470,\"idle\":1986265240,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23966110,\"nice\":16540,\"sys\":3497870,\"idle\":1986491280,\"irq\":0}}],\"eventLoopStats\":{\"min\":164,\"max\":30006,\"num\":3,\"sum\":60018}}}"}
2025-03-17T08:53:09.357462+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201589357,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:54:09.381407+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201649381,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2874130432,\"uptime\":2021544.76,\"totalMem\":4018151424,\"loadavg\":[0.04,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24251360,\"nice\":16990,\"sys\":3481540,\"idle\":1986324490,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23966690,\"nice\":16540,\"sys\":3497950,\"idle\":1986550580,\"irq\":0}}],\"eventLoopStats\":{\"min\":152,\"max\":30030,\"num\":3,\"sum\":60023}}}"}
2025-03-17T08:54:09.381553+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201649381,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:54:48.416037+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688415,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:54:48.416846+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201688416,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-page-response' for consumer group='icc-reporting'"}
2025-03-17T08:54:48.416934+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688416,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:54:48.406Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"id\":\"28c9ce55-d7a3-4127-9375-f2c1d44cc899\",\"serviceId\":\"PEP3OTT\",\"serviceName\":\"GUS: Core Database Performance Incident Escalation\",\"serviceLink\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\",\"responderName\":\"Joydeep Lahiri\",\"responderId\":\"P2Z52MQ\",\"responderLink\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\",\"incidentNumber\":\"14036731\",\"incidentTitle\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"incidentDetails\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \",\"incidentLink\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"success\":true,\"pageTimestamp\":\"2025-03-17T08:54:48.351Z\",\"pdData\":{\"incident\":{\"incident_number\":14036731,\"title\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"created_at\":\"2025-03-17T08:54:48Z\",\"updated_at\":\"2025-03-17T08:54:48Z\",\"status\":\"triggered\",\"incident_key\":\"a3882ea7fd6542a6b5254eb2b6764238\",\"service\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"assignments\":[{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P2Z52MQ\",\"type\":\"user_reference\",\"summary\":\"Joydeep Lahiri\",\"self\":\"https://api.pagerduty.com/users/P2Z52MQ\",\"html_url\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"PS9IDMM\",\"type\":\"user_reference\",\"summary\":\"SAGAR MAHADIK\",\"self\":\"https://api.pagerduty.com/users/PS9IDMM\",\"html_url\":\"https://salesforce.pagerduty.com/users/PS9IDMM\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P6FNWRE\",\"type\":\"user_reference\",\"summary\":\"Sai Kumar Papini\",\"self\":\"https://api.pagerduty.com/users/P6FNWRE\",\"html_url\":\"https://salesforce.pagerduty.com/users/P6FNWRE\"}}],\"assigned_via\":\"escalation_policy\",\"last_status_change_at\":\"2025-03-17T08:54:48Z\",\"resolved_at\":null,\"first_trigger_log_entry\":{\"id\":\"R9ETU43DRILD1UPIF7FW3AE89F\",\"type\":\"trigger_log_entry_reference\",\"summary\":\"Triggered through the website.\",\"self\":\"https://api.pagerduty.com/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\"},\"alert_counts\":{\"all\":0,\"triggered\":0,\"resolved\":0},\"is_mergeable\":true,\"incident_type\":{\"name\":\"incident_default\"},\"escalation_policy\":{\"id\":\"PRLBQP7\",\"type\":\"escalation_policy_reference\",\"summary\":\"Core Database Performance Escalation Policy\",\"self\":\"https://api.pagerduty.com/escalation_policies/PRLBQP7\",\"html_url\":\"https://salesforce.pagerduty.com/escalation_policies/PRLBQP7\"},\"teams\":[{\"id\":\"P39TTHN\",\"type\":\"team_reference\",\"summary\":\"GUS: Core Database Performance\",\"self\":\"https://api.pagerduty.com/teams/P39TTHN\",\"html_url\":\"https://salesforce.pagerduty.com/teams/P39TTHN\"}],\"impacted_services\":[{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"}],\"pending_actions\":[{\"type\":\"escalate\",\"at\":\"2025-03-17T09:04:48Z\"},{\"type\":\"resolve\",\"at\":\"2025-03-17T12:54:48Z\"}],\"acknowledgements\":[],\"basic_alert_grouping\":null,\"alert_grouping\":null,\"last_status_change_by\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"priority\":null,\"incidents_responders\":[],\"responder_requests\":[],\"subscriber_requests\":[],\"urgency\":\"high\",\"id\":\"Q1UWB85OQMTPTR\",\"type\":\"incident\",\"summary\":\"[#14036731] Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"self\":\"https://api.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"body\":{\"details\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \"}}},\"responderEmail\":\"jlahiri@salesforce.com\"}}"}
2025-03-17T08:54:48.416935+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688416,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-response\"}}"}
2025-03-17T08:54:48.416974+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688416,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-page-response\"}}"}
2025-03-17T08:54:48.417114+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:54:48.417115+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-page-response"}
2025-03-17T08:54:48.417118+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-page-response is now being processed."}
2025-03-17T08:54:48.417254+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T08:54:48.417821+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId ODAAAAeWWAB and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,112,97,103,101,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,56,58,53,52,58,52,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,111,110,45,99,97,108,108]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T08:54:48.406Z\\\",\\\"user\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"icc-73104263\\\"},\\\"subject\\\":\\\"APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"description\\\":\\\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\\\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\\\n\\\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\\\",\\\"detectedTime\\\":\\\"2025-03-17T08:06:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T07:53:00.000Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hfi-qyjv-nzz\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\\\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73104263\\\",\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/m6mBA2aDHeqm\\\",\\\"engagedAt\\\":\\\"2025-03-17T08:12:58.310Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T08:19:09.243Z\\\"},\\\"eventObj\\\":{\\\"id\\\":\\\"28c9ce55-d7a3-4127-9375-f2c1d44cc899\\\",\\\"serviceId\\\":\\\"PEP3OTT\\\",\\\"serviceName\\\":\\\"GUS: Core Database Performance Incident Escalation\\\",\\\"serviceLink\\\":\\\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\\\",\\\"responderName\\\":\\\"Joydeep Lahiri\\\",\\\"responderId\\\":\\\"P2Z52MQ\\\",\\\"responderLink\\\":\\\"https://salesforce.pagerduty.com/users/P2Z52MQ\\\",\\\"incidentNumber\\\":\\\"14036731\\\",\\\"incidentTitle\\\":\\\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"incidentDetails\\\":\\\"Please join this active incident:\\\\nTeam: Site Reliability\\\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\\nSeverity: Sev2\\\\nSlack Channel: #icc-73104263\\\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\\\nAdditional Joining Info: Join By Phone:\\\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\\\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\\\n\\\\nAdditional Details: \\\",\\\"incidentLink\\\":\\\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\\\",\\\"success\\\":true,\\\"pageTimestamp\\\":\\\"2025-03-17T08:54:48.351Z\\\",\\\"pdData\\\":{\\\"incident\\\":{\\\"incident_number\\\":14036731,\\\"title\\\":\\\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"description\\\":\\\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"created_at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"updated_at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"status\\\":\\\"triggered\\\",\\\"incident_key\\\":\\\"a3882ea7fd6542a6b5254eb2b6764238\\\",\\\"service\\\":{\\\"id\\\":\\\"PEP3OTT\\\",\\\"type\\\":\\\"service_reference\\\",\\\"summary\\\":\\\"GUS: Core Database Performance Incident Escalation\\\",\\\"self\\\":\\\"https://api.pagerduty.com/services/PEP3OTT\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\\\"},\\\"assignments\\\":[{\\\"at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"assignee\\\":{\\\"id\\\":\\\"P2Z52MQ\\\",\\\"type\\\":\\\"user_reference\\\",\\\"summary\\\":\\\"Joydeep Lahiri\\\",\\\"self\\\":\\\"https://api.pagerduty.com/users/P2Z52MQ\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/users/P2Z52MQ\\\"}},{\\\"at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"assignee\\\":{\\\"id\\\":\\\"PS9IDMM\\\",\\\"type\\\":\\\"user_reference\\\",\\\"summary\\\":\\\"SAGAR MAHADIK\\\",\\\"self\\\":\\\"https://api.pagerduty.com/users/PS9IDMM\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/users/PS9IDMM\\\"}},{\\\"at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"assignee\\\":{\\\"id\\\":\\\"P6FNWRE\\\",\\\"type\\\":\\\"user_reference\\\",\\\"summary\\\":\\\"Sai Kumar Papini\\\",\\\"self\\\":\\\"https://api.pagerduty.com/users/P6FNWRE\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/users/P6FNWRE\\\"}}],\\\"assigned_via\\\":\\\"escalation_policy\\\",\\\"last_status_change_at\\\":\\\"2025-03-17T08:54:48Z\\\",\\\"resolved_at\\\":null,\\\"first_trigger_log_entry\\\":{\\\"id\\\":\\\"R9ETU43DRILD1UPIF7FW3AE89F\\\",\\\"type\\\":\\\"trigger_log_entry_reference\\\",\\\"summary\\\":\\\"Triggered through the website.\\\",\\\"self\\\":\\\"https://api.pagerduty.com/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\\\"},\\\"alert_counts\\\":{\\\"all\\\":0,\\\"triggered\\\":0,\\\"resolved\\\":0},\\\"is_mergeable\\\":true,\\\"incident_type\\\":{\\\"name\\\":\\\"incident_default\\\"},\\\"escalation_policy\\\":{\\\"id\\\":\\\"PRLBQP7\\\",\\\"type\\\":\\\"escalation_policy_reference\\\",\\\"summary\\\":\\\"Core Database Performance Escalation Policy\\\",\\\"self\\\":\\\"https://api.pagerduty.com/escalation_policies/PRLBQP7\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/escalation_policies/PRLBQP7\\\"},\\\"teams\\\":[{\\\"id\\\":\\\"P39TTHN\\\",\\\"type\\\":\\\"team_reference\\\",\\\"summary\\\":\\\"GUS: Core Database Performance\\\",\\\"self\\\":\\\"https://api.pagerduty.com/teams/P39TTHN\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/teams/P39TTHN\\\"}],\\\"impacted_services\\\":[{\\\"id\\\":\\\"PEP3OTT\\\",\\\"type\\\":\\\"service_reference\\\",\\\"summary\\\":\\\"GUS: Core Database Performance Incident Escalation\\\",\\\"self\\\":\\\"https://api.pagerduty.com/services/PEP3OTT\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\\\"}],\\\"pending_actions\\\":[{\\\"type\\\":\\\"escalate\\\",\\\"at\\\":\\\"2025-03-17T09:04:48Z\\\"},{\\\"type\\\":\\\"resolve\\\",\\\"at\\\":\\\"2025-03-17T12:54:48Z\\\"}],\\\"acknowledgements\\\":[],\\\"basic_alert_grouping\\\":null,\\\"alert_grouping\\\":null,\\\"last_status_change_by\\\":{\\\"id\\\":\\\"PEP3OTT\\\",\\\"type\\\":\\\"service_reference\\\",\\\"summary\\\":\\\"GUS: Core Database Performance Incident Escalation\\\",\\\"self\\\":\\\"https://api.pagerduty.com/services/PEP3OTT\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\\\"},\\\"priority\\\":null,\\\"incidents_responders\\\":[],\\\"responder_requests\\\":[],\\\"subscriber_requests\\\":[],\\\"urgency\\\":\\\"high\\\",\\\"id\\\":\\\"Q1UWB85OQMTPTR\\\",\\\"type\\\":\\\"incident\\\",\\\"summary\\\":\\\"[#14036731] Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"self\\\":\\\"https://api.pagerduty.com/incidents/Q1UWB85OQMTPTR\\\",\\\"html_url\\\":\\\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\\\",\\\"body\\\":{\\\"details\\\":\\\"Please join this active incident:\\\\nTeam: Site Reliability\\\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\\\nSeverity: Sev2\\\\nSlack Channel: #icc-73104263\\\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\\\nAdditional Joining Info: Join By Phone:\\\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\\\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\\\n\\\\nAdditional Details: \\\"}}},\\\"responderEmail\\\":\\\"jlahiri@salesforce.com\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T08:54:48.406Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"
2025-03-17T08:54:48.417821+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"id\":\"28c9ce55-d7a3-4127-9375-f2c1d44cc899\",\"serviceId\":\"PEP3OTT\",\"serviceName\":\"GUS: Core Database Performance Incident Escalation\",\"serviceLink\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\",\"responderName\":\"Joydeep Lahiri\",\"responderId\":\"P2Z52MQ\",\"responderLink\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\",\"incidentNumber\":\"14036731\",\"incidentTitle\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"incidentDetails\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \",\"incidentLink\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"success\":true,\"pageTimestamp\":\"2025-03-17T08:54:48.351Z\",\"pdData\":{\"incident\":{\"incident_number\":14036731,\"title\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"created_at\":\"2025-03-17T08:54:48Z\",\"updated_at\":\"2025-03-17T08:54:48Z\",\"status\":\"triggered\",\"incident_key\":\"a3882ea7fd6542a6b5254eb2b6764238\",\"service\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"assignments\":[{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P2Z52MQ\",\"type\":\"user_reference\",\"summary\":\"Joydeep Lahiri\",\"self\":\"https://api.pagerduty.com/users/P2Z52MQ\",\"html_url\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"PS9IDMM\",\"type\":\"user_reference\",\"summary\":\"SAGAR MAHADIK\",\"self\":\"https://api.pagerduty.com/users/PS9IDMM\",\"html_url\":\"https://salesforce.pagerduty.com/users/PS9IDMM\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P6FNWRE\",\"type\":\"user_reference\",\"summary\":\"Sai Kumar Papini\",\"self\":\"https://api.pagerduty.com/users/P6FNWRE\",\"html_url\":\"https://salesforce.pagerduty.com/users/P6FNWRE\"}}],\"assigned_via\":\"escalation_policy\",\"last_status_change_at\":\"2025-03-17T08:54:48Z\",\"resolved_at\":null,\"first_trigger_log_entry\":{\"id\":\"R9ETU43DRILD1UPIF7FW3AE89F\",\"type\":\"trigger_log_entry_reference\",\"summary\":\"Triggered through the website.\",\"self\":\"https://api.pagerduty.com/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\"},\"alert_counts\":{\"all\":0,\"triggered\":0,\"resolved\":0},\"is_mergeable\":true,\"incident_type\":{\"name\":\"incident_default\"},\"escalation_policy\":{\"id\":\"PRLBQP7\",\"type\":\"escalation_policy_reference\",\"summary\":\"Core Database Performance Escalation Policy\",\"self\":\"https://api.pagerduty.com/escalation_policies/PRLBQP7\",\"html_url\":\"https://salesforce.pagerduty.com/escalation_policies/PRLBQP7\"},\"teams\":[{\"id\":\"P39TTHN\",\"type\":\"team_reference\",\"summary\":\"GUS: Core Database Performance\",\"self\":\"https://api.pagerduty.com/teams/P39TTHN\",\"html_url\":\"https://salesforce.pagerduty.com/teams/P39TTHN\"}],\"impacted_services\":[{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"}],\"pending_actions\":[{\"type\":\"escalate\",\"at\":\"2025-03-17T09:04:48Z\"},{\"type\":\"resolve\",\"at\":\"2025-03-17T12:54:48Z\"}],\"acknowledgements\":[],\"basic_alert_grouping\":null,\"alert_grouping\":null,\"last_status_change_by\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"priority\":null,\"incidents_responders\":[],\"responder_requests\":[],\"subscriber_requests\":[],\"urgency\":\"high\",\"id\":\"Q1UWB85OQMTPTR\",\"type\":\"incident\",\"summary\":\"[#14036731] Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"self\":\"https://api.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"body\":{\"details\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \"}}},\"responderEmail\":\"jlahiri@salesforce.com\"}},\"encoder\":\"json\",\"eventType\":\"icc-page-response\",\"payload\":{\"createdAt\":\"2025-03-17T08:54:48.406Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"id\":\"28c9ce55-d7a3-4127-9375-f2c1d44cc899\",\"serviceId\":\"PEP3OTT\",\"serviceName\":\"GUS: Core Database Performance Incident Escalation\",\"serviceLink\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\",\"responderName\":\"Joydeep Lahiri\",\"responderId\":\"P2Z52MQ\",\"responderLink\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\",\"incidentNumber\":\"14036731\",\"incidentTitle\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"incidentDetails\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \",\"incidentLink\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"success\":true,\"pageTimestamp\":\"2025-03-17T08:54:48.351Z\",\"pdData\":{\"incident\":{\"incident_number\":14036731,\"title\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\
2025-03-17T08:54:48.417821+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ",\"description\":\"Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"created_at\":\"2025-03-17T08:54:48Z\",\"updated_at\":\"2025-03-17T08:54:48Z\",\"status\":\"triggered\",\"incident_key\":\"a3882ea7fd6542a6b5254eb2b6764238\",\"service\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"assignments\":[{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P2Z52MQ\",\"type\":\"user_reference\",\"summary\":\"Joydeep Lahiri\",\"self\":\"https://api.pagerduty.com/users/P2Z52MQ\",\"html_url\":\"https://salesforce.pagerduty.com/users/P2Z52MQ\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"PS9IDMM\",\"type\":\"user_reference\",\"summary\":\"SAGAR MAHADIK\",\"self\":\"https://api.pagerduty.com/users/PS9IDMM\",\"html_url\":\"https://salesforce.pagerduty.com/users/PS9IDMM\"}},{\"at\":\"2025-03-17T08:54:48Z\",\"assignee\":{\"id\":\"P6FNWRE\",\"type\":\"user_reference\",\"summary\":\"Sai Kumar Papini\",\"self\":\"https://api.pagerduty.com/users/P6FNWRE\",\"html_url\":\"https://salesforce.pagerduty.com/users/P6FNWRE\"}}],\"assigned_via\":\"escalation_policy\",\"last_status_change_at\":\"2025-03-17T08:54:48Z\",\"resolved_at\":null,\"first_trigger_log_entry\":{\"id\":\"R9ETU43DRILD1UPIF7FW3AE89F\",\"type\":\"trigger_log_entry_reference\",\"summary\":\"Triggered through the website.\",\"self\":\"https://api.pagerduty.com/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR/log_entries/R9ETU43DRILD1UPIF7FW3AE89F\"},\"alert_counts\":{\"all\":0,\"triggered\":0,\"resolved\":0},\"is_mergeable\":true,\"incident_type\":{\"name\":\"incident_default\"},\"escalation_policy\":{\"id\":\"PRLBQP7\",\"type\":\"escalation_policy_reference\",\"summary\":\"Core Database Performance Escalation Policy\",\"self\":\"https://api.pagerduty.com/escalation_policies/PRLBQP7\",\"html_url\":\"https://salesforce.pagerduty.com/escalation_policies/PRLBQP7\"},\"teams\":[{\"id\":\"P39TTHN\",\"type\":\"team_reference\",\"summary\":\"GUS: Core Database Performance\",\"self\":\"https://api.pagerduty.com/teams/P39TTHN\",\"html_url\":\"https://salesforce.pagerduty.com/teams/P39TTHN\"}],\"impacted_services\":[{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"}],\"pending_actions\":[{\"type\":\"escalate\",\"at\":\"2025-03-17T09:04:48Z\"},{\"type\":\"resolve\",\"at\":\"2025-03-17T12:54:48Z\"}],\"acknowledgements\":[],\"basic_alert_grouping\":null,\"alert_grouping\":null,\"last_status_change_by\":{\"id\":\"PEP3OTT\",\"type\":\"service_reference\",\"summary\":\"GUS: Core Database Performance Incident Escalation\",\"self\":\"https://api.pagerduty.com/services/PEP3OTT\",\"html_url\":\"https://salesforce.pagerduty.com/service-directory/PEP3OTT\"},\"priority\":null,\"incidents_responders\":[],\"responder_requests\":[],\"subscriber_requests\":[],\"urgency\":\"high\",\"id\":\"Q1UWB85OQMTPTR\",\"type\":\"incident\",\"summary\":\"[#14036731] Incident: APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"self\":\"https://api.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"html_url\":\"https://salesforce.pagerduty.com/incidents/Q1UWB85OQMTPTR\",\"body\":{\"details\":\"Please join this active incident:\\nTeam: Site Reliability\\nSubject: APT Degradation on USA662s, core2, aws-prod5-uswest2\\nSeverity: Sev2\\nSlack Channel: #icc-73104263\\nBridge URL: https://meet.google.com/hfi-qyjv-nzz\\nAdditional Joining Info: Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\\nAdditional Details: \"}}},\"responderEmail\":\"jlahiri@salesforce.com\"}}}"}
2025-03-17T08:54:48.417836+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: ODAAAAeWWAB"}
2025-03-17T08:54:48.417836+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-page-response and docId ODAAAAeWWAB"}
2025-03-17T08:54:48.420152+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201688417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handlePageResponse for id 8314687c-abb3-4165-bbbf-414b12112a6b and docId ODAAAAeWWAB"}
2025-03-17T08:56:25.584186+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T08:56:25.584478+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T08:56:25.584514+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T08:56:25.333Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T08:19:09.243Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"9f3c6baf-2290-4a80-949f-3b68059651e4\",\"sequenceNumber\":3}}"}
2025-03-17T08:56:25.584563+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:56:25.584590+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T08:56:25.584706+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:56:25.584706+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response"}
2025-03-17T08:56:25.584738+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T08:56:25.584739+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201785584,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-internal-comms-response correctly routed"}
2025-03-17T08:57:09.449448+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201829449,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2876907520,\"uptime\":2021724.83,\"totalMem\":4018151424,\"loadavg\":[0.06,0.04,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24252470,\"nice\":16990,\"sys\":3481740,\"idle\":1986502130,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23967720,\"nice\":16540,\"sys\":3498190,\"idle\":1986728380,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29874,\"num\":21,\"sum\":60019}}}"}
2025-03-17T08:57:09.449506+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201829449,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T08:58:09.477439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742201889477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2876907520,\"uptime\":2021784.86,\"totalMem\":4018151424,\"loadavg\":[0.02,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24253000,\"nice\":16990,\"sys\":3481810,\"idle\":1986561390,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23968030,\"nice\":16540,\"sys\":3498270,\"idle\":1986787660,\"irq\":0}}],\"eventLoopStats\":{\"min\":126,\"max\":30003,\"num\":3,\"sum\":60024}}}"}
2025-03-17T08:58:09.477517+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742201889477,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:04:09.597457+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202249597,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884173824,\"uptime\":2022144.98,\"totalMem\":4018151424,\"loadavg\":[0.09,0.07,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24255750,\"nice\":16990,\"sys\":3482340,\"idle\":1986917040,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23971030,\"nice\":16540,\"sys\":3498720,\"idle\":1987142940,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29925,\"num\":23,\"sum\":60018}}}"}
2025-03-17T09:04:09.597708+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202249597,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:08:09.701472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202489701,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2883194880,\"uptime\":2022385.08,\"totalMem\":4018151424,\"loadavg\":[0.05,0.09,0.04],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24257700,\"nice\":16990,\"sys\":3482660,\"idle\":1987154090,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23972640,\"nice\":16540,\"sys\":3499040,\"idle\":1987379930,\"irq\":0}}],\"eventLoopStats\":{\"min\":52,\"max\":30030,\"num\":3,\"sum\":60015}}}"}
2025-03-17T09:08:09.701719+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202489701,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:08:45.895847+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525894,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T09:08:45.895855+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-request' for consumer group='icc-reporting'"}
2025-03-17T09:08:45.895883+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T09:08:45.855Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"638f1c4c-ceb8-43a7-baab-05a43a5db340\",\"allInternalCommsReport\":[{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":12,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"8 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"We are proceeding with the customer (Schneider) side mitigation, where they refresh their code and flush their CDN one country at a time. This takes longer than anticipated, but we are seeing improving numbers. We are still impacted. Secondary swimlane, Site Switch is in stand by.\",\"sentAt\":\"2025-03-17T08:50:52.071Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":11,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"Signature support team is in call with customer snyder electric to get their configs updated with correct uris.\\nMeanwhile site switching is considered to move other customers to IA5 live agent to mitigate the issue. Awaiting EVP approval for the same\"},\"sentAt\":\"2025-03-17T07:46:22.866Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":10,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U02L4D3SB9A\",\"sentBy\":\"Priyanka Sharma\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Snyder electric misconfiguration is impacting other customer in the same super pod.\",\"actions\":\"We are trying to reach out to the customer(Snyder electric) to change their configuration but its not feasible to block or change the configuration.So we are trying to get other customers out of impact by having a siteswitch.\"},\"sentAt\":\"2025-03-17T07:10:24.160Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is 
2025-03-17T09:08:45.895883+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR
2025-03-17T09:08:45.895883+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"e13ksn-b7yyau-o6g0p-cur3me-1742202525313\",\"client\":\"icc-ui\"}"}
2025-03-17T09:08:45.895886+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T09:08:45.895887+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-request\"}}"}
2025-03-17T09:08:45.895888+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:08:45.895888+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-request"}
2025-03-17T09:08:45.895888+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-internal-comms-request is now being processed."}
2025-03-17T09:08:45.895889+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525895,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525896,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId QWCAAANuCF5 and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,116,101,114,110,97,108,45,99,111,109,109,115,45,114,101,113,117,101,115,116]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,57,58,48,56,58,52,53,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T09:08:45.855Z\\\",\\\"user\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J0HPF5H9\\\",\\\"name\\\":\\\"icc-73095026\\\"},\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"description\\\":\\\"CP SRE Incident\\\\n\\\\nCase created using .case by @Priyanka Sharma in #sr for NA208\\\",\\\"detectedTime\\\":\\\"2025-03-17T00:25:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\\\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\\\n\\\"},\\\"roles\\\":[{\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"02badda0-dae7-4498-93ec-ab6193b2d8a1\\\",\\\"email\\\":\\\"sitereliability@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"SiteReliability\\\",\\\"firstName\\\":null,\\\"lastName\\\":null,\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"00590000000DrNxAAK\\\",\\\"slackUserId\\\":null,\\\"createdBy\\\":\\\"jgoble@salesforce.com\\\",\\\"modifiedBy\\\":\\\"jgoble@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-11-14T22:00:55.527Z\\\",\\\"modifiedAt\\\":\\\"2022-11-14T22:00:55.527Z\\\"},\\\"id\\\":\\\"80955e31-ab05-4427-add9-3a093398312f\\\",\\\"name\\\":\\\"IncidentCommander\\\",\\\"displayName\\\":\\\"Incident Commander\\\",\\\"user\\\":{\\\"id\\\":\\\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\\\",\\\"email\\\":\\\"psharma2@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Priyanka Sharma\\\",\\\"firstName\\\":\\\"Priyanka\\\",\\\"lastName\\\":\\\"Sharma\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005EE000001oiQHYAY\\\",\\\"slackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"createdBy\\\":\\\"icc@aloha.sso\\\",\\\"modifiedBy\\\":\\\"psharma2@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-03-03T06:58:10.326Z\\\",\\\"modifiedAt\\\":\\\"2022-03-03T06:59:00.333Z\\\"},\\\"externalGusId\\\":null}],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73095026\\\",\\\"caseRecordId\\\":\\\"500EE00001Ty0FFYAZ\\\",\\\"servicesImpacted\\\":[\\\"Core 1P\\\"],\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/M8S4Ap02KbIu\\\",\\\"engagedAt\\\":\\\"2025-03-17T00:26:23.518Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T00:38:13.514Z\\\"},\\\"eventObj\\\":{\\\"internalComms\\\":{\\\"sequenceNumber\\\":13,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WV8TM5WP5\\\",\\\"sentBy\\\":\\\"Krisztian Zsibrek\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"9 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\\\",\\\"sentAt\\\":\\\"2025-03-17T09:08:45.461Z\\\",\\\"swimlaneUpdates\\\":[],\\\"slackChannelIds\\\":[\\\"C05U9CBHX8D\\\",\\\"C05T2A3C319\\\"],\\\"emailAddresses\\\":[\\\"gsm@salesforce.com\\\",\\\"sitereliability@salesforce.com\\\",\\\"techcustomerescalation@salesforce.com\\\"]},\\\"internalCommsId\\\":\\\"638f1c4c-ceb8-43a7-baab-05a43a5db340\\\",\\\"allInternalCommsReport\\\":[{\\\"sequenceNumber\\\":13,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WV8TM5WP5\\\",\\\"sentBy\\\":\\\"Krisztian Zsibrek\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"9 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\\\",\\\"sentAt\\\":\\\"2025-03-17T09:08:45.461Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":12,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WV8TM5WP5\\\",\\\"sentBy\\\":\\\"Krisztian Zsibrek\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"8 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":\\\"We are proceeding with the customer (Schneider) side mitigation, where they refresh their code and flush their CDN one country at a time. This takes longer than anticipated, but we are seeing improving numbers. We are still impacted. Secondary swimlane, Site Switch is in stand by.\\\",\\\"sentAt\\\":\\\"2025-03-17T08:50:52.071Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":11,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"7 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\\\",\\\"actions\\\":\\\"Signature support team is in call with customer snyder electric to get their configs updated with correct uris.\\\\nMeanwhile site switching is considered to move other customers to IA5 live agent to mitigate the issue. Awaiting EVP approval for the same\\\"},\\\"sentAt\\\":\\\"2025-03-17T07:46:22.866Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":10,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InPro
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] gress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"sentBy\\\":\\\"Priyanka Sharma\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"7 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Snyder electric misconfiguration is impacting other customer in the same super pod.\\\",\\\"actions\\\":\\\"We are trying to reach out to the customer(Snyder electric) to change their configuration but its not feasible to block or change the configuration.So we are trying to get other customers out of impact by having a siteswitch.\\\"},\\\"sentAt\\\":\\\"2025-03-17T07:10:24.160Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":9,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"6 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\\\n\\\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\\\",\\\"actions\\\":\\\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\\\n\\\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\\\"},\\\"sentAt\\\":\\\"2025-03-17T06:27:18.054Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":8,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"WAS176048\\\",\\\"sentBy\\\":\\\"Tuan Nguyen\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"5 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"Omnichat team advises that they're seeing a customer \\\\\\\"Schneider electric\\\\\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\\\n\\\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\\\",\\\"needs\\\":\\\"Support will create an outbound case for Schneider electric.\\\"},\\\"sentAt\\\":\\\"2025-03-17T05:55:49.276Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":7,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"5 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\\\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\\\nInvestigation and mitigation steps continue to happen however.\\\"},\\\"sentAt\\\":\\\"2025-03-17T05:14:27.661Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":6,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"4 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\\\",\\\"actions\\\":\\\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\\\"},\\\"sentAt\\\":\\\"2025-03-17T04:39:02.111Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":5,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\\\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\\\"},\\\"sentAt\\\":\\\"2025-03-17T03:54:10.722Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":4,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"3 hours\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\\",\\\"actions\\\":\\\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificat
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] es on the new hosts. The discussions around this are still happening considering the pros and cons.\\\"},\\\"sentAt\\\":\\\"2025-03-17T03:06:01.346Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":3,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"112 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\\\",\\\"actions\\\":\\\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\\\"},\\\"sentAt\\\":\\\"2025-03-17T02:17:40.848Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":2,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"60 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\\\",\\\"actions\\\":\\\"Omni team is investigating the causes for latency.\\\\nHeap dumps are being collected for the hosts to analyse.\\\\nRyan who actually owns the service is paged to investigate further\\\"},\\\"sentAt\\\":\\\"2025-03-17T01:25:49.250Z\\\",\\\"swimlaneUpdates\\\":[]},{\\\"sequenceNumber\\\":1,\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"InProgress\\\",\\\"description\\\":\\\"\\\",\\\"bridgeUrl\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"sentBySlackUserId\\\":\\\"U0329QR7H0W\\\",\\\"sentBy\\\":\\\"Jagruthi Akkaldevi\\\",\\\"cloud\\\":\\\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactEndTime\\\":\\\"\\\",\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"duration\\\":\\\"14 minutes\\\",\\\"incidentCommander\\\":\\\"Priyanka Sharma\\\",\\\"incidentCommanderSlackUserId\\\":\\\"U02L4D3SB9A\\\",\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"content\\\":{\\\"conditions\\\":\\\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\\\",\\\"actions\\\":\\\"Investigating the issue and impact radius is under discussion\\\"},\\\"sentAt\\\":\\\"2025-03-17T00:40:19.849Z\\\",\\\"swimlaneUpdates\\\":[]}]},\\\"requestId\\\":\\\"e13ksn-b7yyau-o6g0p-cur3me-1742202525313\\\",\\\"client\\\":\\\"icc-ui\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T09:08:45.855Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"638f1c4c-ceb8-43a7-baab-05a43a5db340\",\"allInternalCommsReport\":[{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":12,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"8 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"We are proceeding with the customer (Schneid
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] er) side mitigation, where they refresh their code and flush their CDN one country at a time. This takes longer than anticipated, but we are seeing improving numbers. We are still impacted. Secondary swimlane, Site Switch is in stand by.\",\"sentAt\":\"2025-03-17T08:50:52.071Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":11,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"Signature support team is in call with customer snyder electric to get their configs updated with correct uris.\\nMeanwhile site switching is considered to move other customers to IA5 live agent to mitigate the issue. Awaiting EVP approval for the same\"},\"sentAt\":\"2025-03-17T07:46:22.866Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":10,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U02L4D3SB9A\",\"sentBy\":\"Priyanka Sharma\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Snyder electric misconfiguration is impacting other customer in the same super pod.\",\"actions\":\"We are trying to reach out to the customer(Snyder electric) to change their configuration but its not feasible to block or change the configuration.So we are trying to get other customers out of impact by having a siteswitch.\"},\"sentAt\":\"2025-03-17T07:10:24.160Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"20
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] 25-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"e13ksn-b7yyau-o6g0p-cur3me-1742202525313\",\"client\":\"icc-ui\"},\"encoder\":\"json\",\"eventType\":\"icc-internal-comms-request\",\"payload\":{\"createdAt\":\"2025-03-17T09:08:45.855Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"internalComms\":{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[],\"slackChannelIds\":[\"C05U9CBHX8D\",\"C05T2A3C319\"],\"emailAddresses\":[\"gsm@salesforce.com\",\"sitereliability@salesforce.com\",\"techcustomerescalation@salesforce.com\"]},\"internalCommsId\":\"638f1c4c-ceb8-43a7-baab-05a43a5db340\",\"allInternalCommsReport\":[{\"sequenceNumber\":13,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"9 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"Customer side mitigation was not adequate and we decided to proceed with an emergency site switch failover. NA208 is off-peak, customer impact should be minimal. ETA is 30 minutes.\",\"sentAt\":\"2025-03-17T09:08:45.461Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":12,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WV8TM5WP5\",\"sentBy\":\"Krisztian Zsibrek\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"8 hours\",\"i
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ncidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":\"We are proceeding with the customer (Schneider) side mitigation, where they refresh their code and flush their CDN one country at a time. This takes longer than anticipated, but we are seeing improving numbers. We are still impacted. Secondary swimlane, Site Switch is in stand by.\",\"sentAt\":\"2025-03-17T08:50:52.071Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":11,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"Signature support team is in call with customer snyder electric to get their configs updated with correct uris.\\nMeanwhile site switching is considered to move other customers to IA5 live agent to mitigate the issue. Awaiting EVP approval for the same\"},\"sentAt\":\"2025-03-17T07:46:22.866Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":10,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U02L4D3SB9A\",\"sentBy\":\"Priyanka Sharma\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"7 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Snyder electric misconfiguration is impacting other customer in the same super pod.\",\"actions\":\"We are trying to reach out to the customer(Snyder electric) to change their configuration but its not feasible to block or change the configuration.So we are trying to get other customers out of impact by having a siteswitch.\"},\"sentAt\":\"2025-03-17T07:10:24.160Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":9,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"6 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel.\\n\\nLater confirmed that Customer Schneider (orgID 00DA0000000abSm)'s has a  broken implementation which is causing a flood of requests post instance refresh to the LAP servers (SP4 IA4 LAPP cluster C1). This is the reason for customers impacting on NA208 as well.\",\"actions\":\"SL 1: We have paged network teams to engage for a potential block against Schneider org. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\\n\\nSL2: A support outreach to Schneider to have them fix their hardcoding post instant refresh.\"},\"sentAt\":\"2025-03-17T06:27:18.054Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":8,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"WAS176048\",\"sentBy\":\"Tuan Nguyen\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"Omnichat team advises that they're seeing a customer \\\"Schneider electric\\\" (pend orgID) is not handling the visitor in their switch server. Customer implementation is creating new cache keys which is potentially causing increase unneeded traffic and errors on the memcache. \\n\\nCurrent path will look to block or ask customers to update LA services post Instance Refresh.\",\"needs\":\"Support will create an outbound case for Schneider electric.\"},\"sentAt\":\"2025-03-17T05:55:49.276Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":7,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"5 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"As the next resort of mitigation, lapp hosts are being restarted as there are memcache errors observed. \\nMeanwhile customer was asked to enable Enhanced Omni from messaging setting and they did. With enhanced Omni enabled, the chat is working as expected.\\nInvestigation and mitigation steps continue to happen however.\"},\"sentAt\":\"2025-03-17T05:14:27.661Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":6,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"4 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is still ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\\nThe mitigation steps taken are not fully effective and investigation is resumed for alternative solutions\",\"actions\":\"The IDB changes could fix the certificate error but not recover the instance performance unfortunately. Investigation is resumed form there for further solution. Customer side logs are being analysed and also heap/thread dumps are collected for further investigation. No ETA and no path to green as of now.\"},\"sentAt\":\"2025-03-17T04:39:02.111Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":5,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view \\nIDB changes are made with EVP approval as part of mitigation. Solution is being monitored now.\"},\"sentAt\":\"2025-03-17T03:54:10.722Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":4,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hs
2025-03-17T09:08:45.896650+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] u\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"3 hours\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel. The certificates on few hosts after instance refresh are not accurate and that is causing the latency.\",\"actions\":\"The action plan now is to redirect the customers to the old instance with appropriate certs through instance refresh or correct the certificates on the new hosts. The discussions around this are still happening considering the pros and cons.\"},\"sentAt\":\"2025-03-17T03:06:01.346Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":3,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"112 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Certificate error on certain instances causing the issue. The mitigation needs IDB changes to fall back to old instances with right certificates. EBF discussions are going on for the IDB changes\"},\"sentAt\":\"2025-03-17T02:17:40.848Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":2,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"60 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"Incident due to OMNI agent performance degradation specifically on one instance na208 is ongoing. Service owners reported the issue through sr slack channel.\",\"actions\":\"Omni team is investigating the causes for latency.\\nHeap dumps are being collected for the hosts to analyse.\\nRyan who actually owns the service is paged to investigate further\"},\"sentAt\":\"2025-03-17T01:25:49.250Z\",\"swimlaneUpdates\":[]},{\"sequenceNumber\":1,\"severity\":\"Sev2\",\"status\":\"InProgress\",\"description\":\"\",\"bridgeUrl\":\"https://meet.google.com/hbh-zkju-hsu\",\"sentBySlackUserId\":\"U0329QR7H0W\",\"sentBy\":\"Jagruthi Akkaldevi\",\"cloud\":\"Customer Centric Reliability Engineering (CCRE) > Capacity and SR\",\"impact\":[\"PerformanceDegradation\"],\"impactEndTime\":\"\",\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"duration\":\"14 minutes\",\"incidentCommander\":\"Priyanka Sharma\",\"incidentCommanderSlackUserId\":\"U02L4D3SB9A\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"team\":\"Site Reliability\",\"content\":{\"conditions\":\"SR received comms in #SR channel about OMNI agent performance degradation specifically on one instance na208.\",\"actions\":\"Investigating the issue and impact radius is under discussion\"},\"sentAt\":\"2025-03-17T00:40:19.849Z\",\"swimlaneUpdates\":[]}]},\"requestId\":\"e13ksn-b7yyau-o6g0p-cur3me-1742202525313\",\"client\":\"icc-ui\"}}"}
2025-03-17T09:08:45.896652+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525896,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: QWCAAANuCF5"}
2025-03-17T09:08:45.896652+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525896,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-internal-comms-request and docId QWCAAANuCF5"}
2025-03-17T09:08:45.896653+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202525896,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleInternalCommsRequest for id 13054fa1-b6e2-45d6-8afe-0c7523384b29 and docId QWCAAANuCF5"}
2025-03-17T09:12:09.773434+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742202729773,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2882756608,\"uptime\":2022625.16,\"totalMem\":4018151424,\"loadavg\":[0,0.03,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24259500,\"nice\":16990,\"sys\":3482900,\"idle\":1987390920,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23974530,\"nice\":16540,\"sys\":3499350,\"idle\":1987616980,\"irq\":0}}],\"eventLoopStats\":{\"min\":63,\"max\":30028,\"num\":3,\"sum\":60008}}}"}
2025-03-17T09:12:09.773482+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202729773,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:13:46.565802+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202826565,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T09:13:46.566541+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742202826566,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T09:18:09.893423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203089893,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2881126400,\"uptime\":2022985.28,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24262630,\"nice\":16990,\"sys\":3483290,\"idle\":1987746240,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23977010,\"nice\":16540,\"sys\":3499740,\"idle\":1987972640,\"irq\":0}}],\"eventLoopStats\":{\"min\":47,\"max\":30014,\"num\":3,\"sum\":60027}}}"}
2025-03-17T09:18:09.893478+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203089893,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:22:09.957400+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203329957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2881060864,\"uptime\":2023225.34,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24264690,\"nice\":16990,\"sys\":3483560,\"idle\":1987982740,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23979130,\"nice\":16540,\"sys\":3500090,\"idle\":1988209260,\"irq\":0}}],\"eventLoopStats\":{\"min\":44,\"max\":30026,\"num\":3,\"sum\":60003}}}"}
2025-03-17T09:22:09.957633+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203329957,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:23:09.985431+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203389985,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2881478656,\"uptime\":2023285.37,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24265030,\"nice\":16990,\"sys\":3483640,\"idle\":1988041970,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23979680,\"nice\":16540,\"sys\":3500180,\"idle\":1988268470,\"irq\":0}}],\"eventLoopStats\":{\"min\":66,\"max\":30030,\"num\":3,\"sum\":60027}}}"}
2025-03-17T09:23:09.985479+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203389985,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:27:13.113951+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203633113,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T09:27:13.114238+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T09:27:13.114268+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T09:27:13.105Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:25:00.000Z\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"origin\":\"Alert\",\"originReference\":\"\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T00:38:13.514Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"3f006e84-4575-4d51-a19c-7d2ae988c7d0\",\"sequenceNumber\":14}}"}
2025-03-17T09:27:13.114320+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:27:13.114348+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:27:13.114585+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:27:13.114586+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response"}
2025-03-17T09:27:13.114587+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T09:27:13.114587+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203633114,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response correctly routed"}
2025-03-17T09:29:10.093423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203750093,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2881515520,\"uptime\":2023645.48,\"totalMem\":4018151424,\"loadavg\":[0.01,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24268290,\"nice\":16990,\"sys\":3484100,\"idle\":1988397460,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23982730,\"nice\":16540,\"sys\":3500680,\"idle\":1988623970,\"irq\":0}}],\"eventLoopStats\":{\"min\":85,\"max\":30027,\"num\":3,\"sum\":60028}}}"}
2025-03-17T09:29:10.093501+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203750093,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:33:10.177562+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742203990177,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886701056,\"uptime\":2023885.56,\"totalMem\":4018151424,\"loadavg\":[0.01,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24269850,\"nice\":16990,\"sys\":3484470,\"idle\":1988634670,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23984390,\"nice\":16540,\"sys\":3500930,\"idle\":1988860750,\"irq\":0}}],\"eventLoopStats\":{\"min\":110,\"max\":30008,\"num\":3,\"sum\":60028}}}"}
2025-03-17T09:33:10.177567+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742203990177,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:34:10.209393+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742204050209,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885992448,\"uptime\":2023945.59,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24270310,\"nice\":16990,\"sys\":3484540,\"idle\":1988694020,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23984740,\"nice\":16540,\"sys\":3501010,\"idle\":1988919940,\"irq\":0}}],\"eventLoopStats\":{\"min\":89,\"max\":30015,\"num\":3,\"sum\":60027}}}"}
2025-03-17T09:34:10.209624+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204050209,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:39:10.297407+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742204350297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885648384,\"uptime\":2024245.68,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24273080,\"nice\":16990,\"sys\":3485000,\"idle\":1988990400,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23987610,\"nice\":16540,\"sys\":3501510,\"idle\":1989216080,\"irq\":0}}],\"eventLoopStats\":{\"min\":76,\"max\":30018,\"num\":3,\"sum\":60019}}}"}
2025-03-17T09:39:10.297458+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204350297,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:42:10.329453+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742204530329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2888151040,\"uptime\":2024425.71,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24274170,\"nice\":16990,\"sys\":3485260,\"idle\":1989167980,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23988690,\"nice\":16540,\"sys\":3501800,\"idle\":1989393930,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30026,\"num\":13,\"sum\":59998}}}"}
2025-03-17T09:42:10.329522+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204530329,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:47:06.503378+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204826503,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T09:47:06.504183+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204826504,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T09:49:10.485454+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742204950485,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885668864,\"uptime\":2024845.87,\"totalMem\":4018151424,\"loadavg\":[0.04,0.03,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24277760,\"nice\":16990,\"sys\":3485840,\"idle\":1989582220,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23992470,\"nice\":16540,\"sys\":3502440,\"idle\":1989808380,\"irq\":0}}],\"eventLoopStats\":{\"min\":146,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T09:49:10.485550+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742204950485,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:51:10.541431+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205070541,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886176768,\"uptime\":2024965.92,\"totalMem\":4018151424,\"loadavg\":[0,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24278910,\"nice\":16990,\"sys\":3485980,\"idle\":1989700700,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23993600,\"nice\":16540,\"sys\":3502610,\"idle\":1989926880,\"irq\":0}}],\"eventLoopStats\":{\"min\":131,\"max\":30002,\"num\":3,\"sum\":60026}}}"}
2025-03-17T09:51:10.541511+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205070541,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:52:07.479675+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205127479,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T09:52:07.480426+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205127480,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T09:54:10.589421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205250589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886434816,\"uptime\":2025145.97,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24280320,\"nice\":16990,\"sys\":3486230,\"idle\":1989878450,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23994810,\"nice\":16540,\"sys\":3502820,\"idle\":1990104640,\"irq\":0}}],\"eventLoopStats\":{\"min\":133,\"max\":30012,\"num\":3,\"sum\":60026}}}"}
2025-03-17T09:54:10.589623+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205250589,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:56:10.645422+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205370645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886463488,\"uptime\":2025266.03,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24281190,\"nice\":16990,\"sys\":3486410,\"idle\":1989996910,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23995950,\"nice\":16540,\"sys\":3502970,\"idle\":1990223160,\"irq\":0}}],\"eventLoopStats\":{\"min\":111,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T09:56:10.645641+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205370645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:57:10.649418+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205430649,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886463488,\"uptime\":2025326.03,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24281630,\"nice\":16990,\"sys\":3486490,\"idle\":1990056210,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23996350,\"nice\":16540,\"sys\":3503070,\"idle\":1990282360,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30030,\"num\":13,\"sum\":59999}}}"}
2025-03-17T09:57:10.649480+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205430649,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:57:21.689643+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205441689,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 13054fa1-b6e2-45d6-8afe-0c7523384b29-incident-info, info: {\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseId\":\"500EE00001Ty0FFYAZ\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"slackChannelId\":\"C08J0HPF5H9\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T00:26:00.000Z\",\"impactEnd\":\"2025-03-17T09:30:00.000Z\",\"servicesImpacted\":\"Core 1P\",\"roles\":{},\"category\":\"Application\",\"rootCause\":\"Configuration\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"caseNumber\":\"73095026\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T09:57:21.690931+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205441690,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-incident-resolved correctly routed"}
2025-03-17T09:58:29.481365+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205509481,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-request correctly routed"}
2025-03-17T09:58:29.532448+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T09:58:29.532675+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T09:58:29.532719+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T09:58:29.522Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"C05U9CBHX8D\",\"deliveryStatus\":true,\"type\":\"SlackChannelId\"},{\"notification\":\"C05T2A3C319\",\"deliveryStatus\":true,\"type\":\"SlackChannelId\"}],\"internalCommsId\":\"1173cec7-a9e4-4cc1-8f01-4dfac580191e\",\"sequenceNumber\":15}}"}
2025-03-17T09:58:29.532773+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:58:29.532830+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:58:29.532956+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:58:29.532956+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response"}
2025-03-17T09:58:29.532959+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T09:58:29.532976+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205509532,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response correctly routed"}
2025-03-17T09:59:10.673449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205550673,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886479872,\"uptime\":2025446.06,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24282210,\"nice\":16990,\"sys\":3486650,\"idle\":1990174840,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23997030,\"nice\":16540,\"sys\":3503240,\"idle\":1990400700,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29875,\"num\":20,\"sum\":59999}}}"}
2025-03-17T09:59:10.673508+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205550673,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:20.969951+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560969,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"request\",\"data\":{\"method\":\"quip\",\"statusCode\":0,\"message\":\"\"}}"}
2025-03-17T09:59:20.970030+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560969,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:20.970203+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560970,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"QuipService.createDocument for id 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 created with URL https://salesforce.quip.com/XugbApggvtpR"}
2025-03-17T09:59:20.970242+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560970,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.setValue key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - value: OZTAAAiPFHp"}
2025-03-17T09:59:20.971314+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8-incident-info, info: {\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"caseId\":\"\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"slackChannelId\":\"C08J2GAMBA7\",\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"Description not found\",\"severity\":\"Sev4\",\"status\":\"New\",\"impact\":\"CurrentlyUnderInvestigation\",\"impactStart\":\"2025-03-17T09:59:15.270Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Services Impacted not found\",\"roles\":{},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Origin not found\",\"originReference\":\"Origin reference URL not found\",\"caseNumber\":\"Case Number not found\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T09:59:20.971548+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"produceEvent with link https://salesforce.quip.com/XugbApggvtpR for event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 and case "}
2025-03-17T09:59:20.971561+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricRequestExecute] - icc-report-created - undefined"}
2025-03-17T09:59:20.971592+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:20.971630+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:20.971710+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:20.971761+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key '36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8' to topic 'incident': {\"createdAt\":\"2025-03-17T09:59:20.971Z\",\"incident\":{\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"slackChannel\":{\"id\":\"C08J2GAMBA7\",\"name\":\"icc-36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\"},\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"\",\"detectedTime\":\"2025-03-17T09:59:15.270Z\",\"severity\":\"Sev4\",\"status\":\"New\",\"origin\":\"\",\"originReference\":\"\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T09:59:15.270Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"scrumTeamId\":\"a00AH000000ikT1YAI\",\"team\":\"Industries Site Reliability Engineering\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/XugbApggvtpR\"}}"}
2025-03-17T09:59:20.971811+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key '36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8' to Kafka topic 'incident'"}
2025-03-17T09:59:20.971814+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricDurationTime] - icc-report-created - undefined"}
2025-03-17T09:59:20.971833+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.10276199877262115\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:20.971866+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.10276199877262115\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:20.971902+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:20.971947+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205560971,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-incident-created correctly routed"}
2025-03-17T09:59:21.262684+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8"}
2025-03-17T09:59:21.262808+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-report-created' for consumer group='icc-reporting'"}
2025-03-17T09:59:21.262810+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T09:59:20.971Z\",\"incident\":{\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"slackChannel\":{\"id\":\"C08J2GAMBA7\",\"name\":\"icc-36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\"},\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"\",\"detectedTime\":\"2025-03-17T09:59:15.270Z\",\"severity\":\"Sev4\",\"status\":\"New\",\"origin\":\"\",\"originReference\":\"\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T09:59:15.270Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"scrumTeamId\":\"a00AH000000ikT1YAI\",\"team\":\"Industries Site Reliability Engineering\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/XugbApggvtpR\"}}"}
2025-03-17T09:59:21.262813+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:21.262882+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-created\"}}"}
2025-03-17T09:59:21.262984+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:21.262985+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-report-created"}
2025-03-17T09:59:21.262986+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205561262,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-report-created is now being processed."}
2025-03-17T09:59:21.263158+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561263,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - value: [object Promise]"}
2025-03-17T09:59:21.263507+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561263,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId OZTAAAiPFHp and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,45,99,114,101,97,116,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,48,57,58,53,57,58,50,48,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,105,110,103]}}],\"key\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T09:59:20.971Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J2GAMBA7\\\",\\\"name\\\":\\\"icc-36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\\\"},\\\"subject\\\":\\\"RCG-GCP Audit compliance login\\\",\\\"description\\\":\\\"\\\",\\\"detectedTime\\\":\\\"2025-03-17T09:59:15.270Z\\\",\\\"severity\\\":\\\"Sev4\\\",\\\"status\\\":\\\"New\\\",\\\"origin\\\":\\\"\\\",\\\"originReference\\\":\\\"\\\",\\\"impact\\\":[\\\"CurrentlyUnderInvestigation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T09:59:15.270Z\\\",\\\"impactEndTime\\\":\\\"\\\",\\\"bridge\\\":{\\\"url\\\":\\\"\\\",\\\"additionalJoiningInfo\\\":\\\"\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"\\\",\\\"rootCause\\\":\\\"\\\",\\\"createdAt\\\":\\\"2025-03-17T09:59:15.270Z\\\",\\\"scrumTeamId\\\":\\\"a00AH000000ikT1YAI\\\",\\\"team\\\":\\\"Industries Site Reliability Engineering\\\",\\\"caseNumber\\\":\\\"\\\",\\\"caseRecordId\\\":\\\"\\\",\\\"servicesImpacted\\\":[],\\\"quipDocLink\\\":\\\"\\\",\\\"engagedAt\\\":\\\"\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"gusExportTimestamp\\\":\\\"\\\"},\\\"eventObj\\\":{\\\"link\\\":\\\"https://salesforce.quip.com/XugbApggvtpR\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T09:59:20.971Z\",\"incident\":{\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"slackChannel\":{\"id\":\"C08J2GAMBA7\",\"name\":\"icc-36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\"},\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"\",\"detectedTime\":\"2025-03-17T09:59:15.270Z\",\"severity\":\"Sev4\",\"status\":\"New\",\"origin\":\"\",\"originReference\":\"\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T09:59:15.270Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"scrumTeamId\":\"a00AH000000ikT1YAI\",\"team\":\"Industries Site Reliability Engineering\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/XugbApggvtpR\"}},\"encoder\":\"json\",\"eventType\":\"icc-report-created\",\"payload\":{\"createdAt\":\"2025-03-17T09:59:20.971Z\",\"incident\":{\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"slackChannel\":{\"id\":\"C08J2GAMBA7\",\"name\":\"icc-36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\"},\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"\",\"detectedTime\":\"2025-03-17T09:59:15.270Z\",\"severity\":\"Sev4\",\"status\":\"New\",\"origin\":\"\",\"originReference\":\"\",\"impact\":[\"CurrentlyUnderInvestigation\"],\"impactStartTime\":\"2025-03-17T09:59:15.270Z\",\"impactEndTime\":\"\",\"bridge\":{\"url\":\"\",\"additionalJoiningInfo\":\"\"},\"roles\":[],\"category\":\"\",\"rootCause\":\"\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"scrumTeamId\":\"a00AH000000ikT1YAI\",\"team\":\"Industries Site Reliability Engineering\",\"caseNumber\":\"\",\"caseRecordId\":\"\",\"servicesImpacted\":[],\"quipDocLink\":\"\",\"engagedAt\":\"\",\"lastAttemptedForceExport\":\"\",\"gusExportTimestamp\":\"\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/XugbApggvtpR\"}}}"}
2025-03-17T09:59:21.263509+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561263,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: OZTAAAiPFHp"}
2025-03-17T09:59:21.263510+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205561263,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-report-created and docId OZTAAAiPFHp"}
2025-03-17T09:59:21.263511+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205561263,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-report-created correctly routed"}
2025-03-17T09:59:23.728058+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205563727,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-gus-response correctly routed"}
2025-03-17T09:59:40.154588+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T09:59:40.154817+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-internal-comms-response' for consumer group='icc-reporting'"}
2025-03-17T09:59:40.154878+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T09:59:40.145Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"notifications\":[{\"notification\":\"gsm@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"sitereliability@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true},{\"notification\":\"techcustomerescalation@salesforce.com\",\"type\":\"EmailAddress\",\"deliveryStatus\":true}],\"internalCommsId\":\"1173cec7-a9e4-4cc1-8f01-4dfac580191e\",\"sequenceNumber\":15}}"}
2025-03-17T09:59:40.154882+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:59:40.154947+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-internal-comms-response\"}}"}
2025-03-17T09:59:40.155028+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T09:59:40.155028+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205580154,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response"}
2025-03-17T09:59:40.155042+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205580155,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-internal-comms-response is not consumed by icc-reporting."}
2025-03-17T09:59:40.155055+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205580155,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-internal-comms-response correctly routed"}
2025-03-17T09:59:47.995014+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205587994,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8-incident-info, info: {\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"caseId\":\"500EE00001TyxbxYAB\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"slackChannelId\":\"C08J2GAMBA7\",\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"Description not found\",\"severity\":\"Sev4\",\"status\":\"InProgress\",\"impact\":\"CurrentlyUnderInvestigation\",\"impactStart\":\"2025-03-17T09:59:15.270Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Services Impacted not found\",\"roles\":{\"Incident Commander\":\"Vamsi Vemula\"},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Origin not found\",\"originReference\":\"Origin reference URL not found\",\"caseNumber\":\"73106543\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T09:59:47.996064+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205587996,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-incident-updated correctly routed"}
2025-03-17T09:59:48.543372+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205588543,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8-incident-info, info: {\"id\":\"36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8\",\"caseId\":\"500EE00001TyxbxYAB\",\"createdAt\":\"2025-03-17T09:59:15.270Z\",\"slackChannelId\":\"C08J2GAMBA7\",\"subject\":\"RCG-GCP Audit compliance login\",\"description\":\"Description not found\",\"severity\":\"Sev4\",\"status\":\"InProgress\",\"impact\":\"CurrentlyUnderInvestigation\",\"impactStart\":\"2025-03-17T09:59:15.270Z\",\"impactEnd\":\"0\",\"servicesImpacted\":\"Services Impacted not found\",\"roles\":{\"Incident Commander\":\"Vamsi Vemula\"},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Origin not found\",\"originReference\":\"Origin reference URL not found\",\"caseNumber\":\"73106543\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T09:59:48.544429+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205588544,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 36929a0f-3861-4ced-9cc2-c8b2f0f8b6f8 - icc-incident-updated correctly routed"}
2025-03-17T10:00:10.689385+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205610689,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886225920,\"uptime\":2025506.07,\"totalMem\":4018151424,\"loadavg\":[0.23,0.08,0.03],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24282700,\"nice\":16990,\"sys\":3486710,\"idle\":1990234040,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23997480,\"nice\":16540,\"sys\":3503340,\"idle\":1990459860,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":17127,\"num\":97,\"sum\":59981}}}"}
2025-03-17T10:00:10.689605+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205610689,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:01:10.709425+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205670709,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886225920,\"uptime\":2025566.09,\"totalMem\":4018151424,\"loadavg\":[0.08,0.07,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24283200,\"nice\":16990,\"sys\":3486780,\"idle\":1990293390,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23998240,\"nice\":16540,\"sys\":3503420,\"idle\":1990519070,\"irq\":0}}],\"eventLoopStats\":{\"min\":107,\"max\":30027,\"num\":3,\"sum\":60017}}}"}
2025-03-17T10:01:10.709479+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205670709,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:02:10.729386+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205730729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886225920,\"uptime\":2025626.11,\"totalMem\":4018151424,\"loadavg\":[0.03,0.05,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24283840,\"nice\":16990,\"sys\":3486850,\"idle\":1990352610,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23998820,\"nice\":16540,\"sys\":3503520,\"idle\":1990578360,\"irq\":0}}],\"eventLoopStats\":{\"min\":117,\"max\":30030,\"num\":3,\"sum\":60019}}}"}
2025-03-17T10:02:10.729576+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205730729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:10.757390+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205790757,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2886238208,\"uptime\":2025686.14,\"totalMem\":4018151424,\"loadavg\":[0.01,0.04,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24284220,\"nice\":16990,\"sys\":3486920,\"idle\":1990411920,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":23999410,\"nice\":16540,\"sys\":3503600,\"idle\":1990637540,\"irq\":0}}],\"eventLoopStats\":{\"min\":2,\"max\":29889,\"num\":7,\"sum\":60025}}}"}
2025-03-17T10:03:10.757501+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205790757,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.347245+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.347809+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.347811+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.336Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}"}
2025-03-17T10:03:58.347814+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:03:58.347814+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:03:58.347815+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.347815+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-deleted"}
2025-03-17T10:03:58.347816+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T10:03:58.347816+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838347,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-deleted correctly routed"}
2025-03-17T10:03:58.369458+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.369662+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-created' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.369740+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.360Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"CP SRE Incident\\n\\nCase created using .case by @Priyanka Sharma in #sr for NA208\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":[],\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}"}
2025-03-17T10:03:58.369757+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:03:58.369780+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:03:58.369874+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.369875+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-created"}
2025-03-17T10:03:58.369899+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-created is not consumed by icc-reporting."}
2025-03-17T10:03:58.369909+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838369,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-created correctly routed"}
2025-03-17T10:03:58.417218+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.417294+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-response' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.417299+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.369Z\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"responses\":[]}"}
2025-03-17T10:03:58.417313+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:03:58.417378+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:03:58.417463+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.417670+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742205838417,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:03:58.618856+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838618,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.619102+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.619195+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.610Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}"}
2025-03-17T10:03:58.619199+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:03:58.619229+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:03:58.619373+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.619374+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-deleted"}
2025-03-17T10:03:58.619374+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T10:03:58.619397+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838619,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-deleted correctly routed"}
2025-03-17T10:03:58.641020+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838640,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.641326+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-created' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.641411+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.632Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"modifiedAt\":\"2025-03-17T00:26:10.551Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"80955e31-ab05-4427-add9-3a093398312f\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"ad04ea4c-799b-4b6e-a952-344bb4b1e4a7\",\"email\":\"psharma2@salesforce.com\",\"active\":true,\"name\":\"Priyanka Sharma\",\"firstName\":\"Priyanka\",\"lastName\":\"Sharma\",\"profile\":\"icc standard user\",\"gusUserId\":\"005EE000001oiQHYAY\",\"slackUserId\":\"U02L4D3SB9A\",\"createdBy\":\"icc@aloha.sso\",\"modifiedBy\":\"psharma2@salesforce.com\",\"createdAt\":\"2022-03-03T06:58:10.326Z\",\"modifiedAt\":\"2022-03-03T06:59:00.333Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":[],\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}"}
2025-03-17T10:03:58.641414+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:03:58.641464+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:03:58.641572+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.641573+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-created"}
2025-03-17T10:03:58.641573+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-created is not consumed by icc-reporting."}
2025-03-17T10:03:58.641594+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838641,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-usd-resource-created correctly routed"}
2025-03-17T10:03:58.645528+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.645871+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-closed' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.645951+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.372Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}"}
2025-03-17T10:03:58.645954+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-closed\"}}"}
2025-03-17T10:03:58.645986+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838645,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-closed\"}}"}
2025-03-17T10:03:58.646098+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.646098+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-incident-closed"}
2025-03-17T10:03:58.646099+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-incident-closed is now being processed."}
2025-03-17T10:03:58.646220+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T10:03:58.646752+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId QWCAAANuCF5 and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,105,110,99,105,100,101,110,116,45,99,108,111,115,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,48,51,58,53,56,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,115,116]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:03:58.372Z\\\",\\\"user\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J0HPF5H9\\\",\\\"name\\\":\\\"icc-73095026\\\"},\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"description\\\":\\\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\\\",\\\"detectedTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Closed\\\",\\\"origin\\\":\\\"Customer - External\\\",\\\"originReference\\\":\\\"--\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T09:30:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\\\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Configuration\\\",\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"closingDetails\\\":{\\\"productTag\\\":\\\"a1aEE000000RZw9YAG\\\",\\\"detectedByServiceOwner\\\":true,\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"proximateCause\\\":\\\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\\\",\\\"customerExperience\\\":\\\"Chat function not working\\\",\\\"immediateResolution\\\":\\\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\\\",\\\"aarQuestions\\\":\\\"- Why were we so slow to implement the solution?\\\",\\\"changes\\\":\\\"\\\",\\\"breakFix\\\":\\\"\\\",\\\"productTagName\\\":\\\"SCRT1 Central\\\",\\\"closedBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"incidentId\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"closingComments\\\":null,\\\"createdAt\\\":\\\"2025-03-17T10:03:58.286Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T10:03:58.286Z\\\",\\\"id\\\":\\\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\\\"},\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73095026\\\",\\\"caseRecordId\\\":\\\"500EE00001Ty0FFYAZ\\\",\\\"servicesImpacted\\\":[\\\"Core 1P\\\"],\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/M8S4Ap02KbIu\\\",\\\"engagedAt\\\":\\\"2025-03-17T00:26:23.518Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T09:57:25.281Z\\\"},\\\"requestId\\\":\\\"mto95-v069v-4hmnt6-g8dra-1742205838144\\\",\\\"client\\\":\\\"icc-ui\\\"}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:03:58.372Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\"
2025-03-17T10:03:58.646752+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ,\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"},\"encoder\":\"json\",\"eventType\":\"icc-incident-closed\",\"payload\":{\"createdAt\":\"2025-03-17T10:03:58.372Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"requestId\":\"mto95-v069v-4hmnt6-g8dra-1742205838144\",\"client\":\"icc-ui\"}}"}
2025-03-17T10:03:58.646753+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: QWCAAANuCF5"}
2025-03-17T10:03:58.646753+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-incident-closed and docId QWCAAANuCF5"}
2025-03-17T10:03:58.646758+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleIncidentClosed for id 13054fa1-b6e2-45d6-8afe-0c7523384b29 and docId QWCAAANuCF5"}
2025-03-17T10:03:58.646758+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: getIncidentInfo, key: 13054fa1-b6e2-45d6-8afe-0c7523384b29-incident-info"}
2025-03-17T10:03:58.647053+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838646,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29-gus-casenumber - value: [object Promise]"}
2025-03-17T10:03:58.693277+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:03:58.693419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-response' for consumer group='icc-reporting'"}
2025-03-17T10:03:58.693423+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:03:58.641Z\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"responses\":[]}"}
2025-03-17T10:03:58.693437+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:03:58.693496+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:03:58.693585+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:03:58.693800+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742205838693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:03:59.548779+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205839548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Roles for  incident: []"}
2025-03-17T10:04:00.357592+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205840357,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 13054fa1-b6e2-45d6-8afe-0c7523384b29-incident-info, info: {\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseId\":\"500EE00001Ty0FFYAZ\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"slackChannelId\":\"C08J0HPF5H9\",\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T00:26:00.000Z\",\"impactEnd\":\"2025-03-17T09:30:00.000Z\",\"servicesImpacted\":\"Core 1P\",\"roles\":{},\"category\":\"Application\",\"rootCause\":\"Configuration\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"caseNumber\":\"73095026\",\"productTag\":\"a1aEE000000RZw9YAG\",\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Krisztian Zsibrek\"}"}
2025-03-17T10:04:00.358616+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205840358,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages Adding Slack messages to docId QWCAAANuCF5..."}
2025-03-17T10:04:01.549955+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841549,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02GXBX8PUZ"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.560634+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841560,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02Q288NF7W"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.567361+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841567,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U01UPBJAV5F"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.598615+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841598,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U032QNNKMT7"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.607692+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841607,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U03LNRS637U"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.612034+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841611,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02DWLVTKB9"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:01.614853+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205841614,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U06Q54CS88M"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:04:02.118995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205842118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages internalSlackMessages for docId QWCAAANuCF5 - [{\"date\":\"1742171172.237659\",\"message\":\"<Incident Command Center> has joined the channel\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171172.746699\",\"message\":\"set the channel topic: :icc-sev2: Sev2 | CP SRE Incident\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171172.925269\",\"message\":\"set the channel description: This is the main *Incident Command Center (ICC)* channel for everyone involved in responding to this incident.\\n\\nVisit *ICC App Home* (<slack://app?team=T01GST6QY0G&amp;id=A02Q50RQQ3B>) to learn more about everything you can do here.\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171174.092279\",\"message\":\"<Priyanka Sharma> has joined the channel\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742171174.361969\",\"message\":\"<HawkEye> has joined the channel\",\"username\":\"HawkEye\",\"replies\":[]},{\"date\":\"1742171174.455449\",\"message\":\"<Fawkes> has joined the channel\",\"username\":\"Fawkes\",\"replies\":[]},{\"date\":\"1742171174.551319\",\"message\":\"<SR-Scheduler> has joined the channel\",\"username\":\"SR-Scheduler\",\"replies\":[]},{\"date\":\"1742171174.647169\",\"message\":\"<ROB> has joined the channel\",\"username\":\"ROB\",\"replies\":[]},{\"date\":\"1742171174.698409\",\"message\":\"set the channel topic: :icc-sev2: Sev2 | CP SRE Incident | <https://meet.google.com/hbh-zkju-hsu>\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171174.739069\",\"message\":\"<Ask Site Switch> has joined the channel\",\"username\":\"Ask Site Switch\",\"replies\":[]},{\"date\":\"1742171174.821749\",\"message\":\"<AIOpsAgent> has joined the channel\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742171175.028399\",\"message\":\"Bridge Added\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171180.437149\",\"message\":\"has renamed the channel from \\\"icc-13054fa1-b6e2-45d6-8afe-0c7523384b29\\\" to \\\"icc-73095026\\\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171184.041029\",\"message\":\"SiteReliability just updated this incident:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171184.044569\",\"message\":\"SiteReliability updated status from *New* to *In Progress*. Let's get to work!\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171184.258939\",\"message\":\"SiteReliability just updated this incident:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171204.823799\",\"message\":\":alarm-clock2: The below reminder is NOW DUE:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171410.629229\",\"message\":\"<@WBFEYNGA1> has joined the channel\",\"username\":\"Anna Marie Kaibigan\",\"replies\":[]},{\"date\":\"1742171459.917189\",\"message\":\"<@WSJQS1AF6> has joined the channel\",\"username\":\"Sahan Dissanayake\",\"replies\":[]},{\"date\":\"1742171501.269909\",\"message\":\"<Jagruthi Akkaldevi> has joined the channel\",\"username\":\"Jagruthi Akkaldevi\",\"replies\":[]},{\"date\":\"1742171784.652589\",\"message\":\":alarm-clock2: The below reminder is NOW DUE:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171889.117299\",\"message\":\"set the channel topic: :icc-sev2: Sev2 | Customers seeing latency on OMNI chat - single pod | <https://meet.google.com/hbh-zkju-hsu>\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171889.451789\",\"message\":\"<Jagruthi Akkaldevi> just updated this incident:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742171969.340969\",\"message\":\"<John Hongrae Lee> has joined the channel\",\"username\":\"John Hongrae Lee\",\"replies\":[]},{\"date\":\"1742171992.871189\",\"message\":\"<@WAS176048> has joined the channel\",\"username\":\"Tuan Nguyen\",\"replies\":[]},{\"date\":\"1742171998.315769\",\"message\":\"<Neha Baghel> has joined the channel\",\"username\":\"Neha Baghel\",\"replies\":[]},{\"date\":\"1742172021.543249\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742172189.029389\",\"message\":\"two simlanes 1) get sources for latency 2) get heapdumps for hosts\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742172419.986799\",\"message\":\"<Sucheta Roy> has joined the channel\",\"username\":\"Sucheta Roy\",\"replies\":[]},{\"date\":\"1742172461.483339\",\"message\":\"<@WK0HU5Z6F> has joined the channel\",\"username\":\"Sandhya Veluri\",\"replies\":[]},{\"date\":\"1742172759.375989\",\"message\":\"<Vineet Kumar> has joined the channel\",\"username\":\"Vineet Kumar\",\"replies\":[]},{\"date\":\"1742172766.169379\",\"message\":\"<@WAU3T5X9S> has joined the channel\",\"username\":\"Sunny Chiu\",\"replies\":[]},{\"date\":\"1742172789.493489\",\"message\":\"<Bhasker Konakanchi> can you please share the start time cc <Jagruthi Akkaldevi>\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742172809.658639\",\"message\":\"<Bhasker Konakanchi> has joined the channel\",\"username\":\"Bhasker Konakanchi\",\"replies\":[]},{\"date\":\"1742172870.660699\",\"message\":\"<@WF8D3GCG7> has joined the channel\",\"username\":\"Guru Balse\",\"replies\":[]},{\"date\":\"1742172972.192309\",\"message\":\"I am looking at <https://moncloud-grafana.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl/d/haPuS7gnz/scrt1-1p-metrics-scrt-devops?orgId=1&from=now-1d&to=now-5m&var-DataCenter=IA4&var-SuperPod=SP5&var-Cluster=lapp5ia41&var-Host=All&var-Scrtkafka_Cluster=scrtkafka5ia41&var-Scrtkafka_Hosts=All&viewPanel=172|https://moncloud-grafana.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl/d/haPuS7gnz/scrt1-1p-metrics-scrt-d[…]ar-Scrtkafka_Hosts=All&viewPanel=172> and can see that the issues started as below\",\"username\":\"Guru Balse\",\"replies\":[],\"attachments\":[{\"name\":\"image.png\",\"url\":\"https://files.slack.com/files-pri/T7T5PNK3P-F08HZKUDMEE/image.png\"}]},{\"date\":\"1742173545.784929\",\"message\":\"There appears to be a pretty clear spike in response time on this instance - <https://monitoring.internal.salesforce.com/argusmvp/#/viewmetricsnew?expression=-72h%3A-0m%3Aservice-cloud-realtime.IA4.SP5.lapp5ia41%3Aapp.chat.request.requestTimer.99thPercentile%3Aavg%3A1m-avg&amp;tab=expression|https://monitoring.internal.salesforce.com/argusmvp/#/viewmetricsnew?expression=-72h%3A-0m%3Aservic[…]r.99thPercentile%3Aavg%3A1m-avg&amp;tab=expression>\",\"username\":\"Jon Hanson\",\"replies\":[{\"date\":\"1742174237.264549\",\"message\":\"Query shows spike in Gack related to <https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=1742174124.119502_4A14D634-95F4-4C30-A7CA-5B6D5306162A|https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=174[…]D634-95F4-4C30-A7CA-5B6D5306162A> Omni availability timeouts\",\"username\":\"Jon Hanson\",\"replies\":[]},{\"date\":\"1742174571.190009\",\"message\":\"<https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=1742174513.119519_4A14D634-95F4-4C30-A7CA-5B6D5306162A|https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=174[…]D634-95F4-4C30-A7CA-5B6D5306162A>\",\"username\":\"Jon Hanson\",\"replies\":[]},{\"date\":\"1742174624.019569\",\"message\":\"both seem to correlate with CUJ flapping timeline, so both appear to lineup with my understanding of impact\",\"username\":\"Jon Hanson\",\"replies\":[]},{\"date\":\"1742174886.413109\",\"message\":\"<@WAQM68L1W>\",\"username\":\"Jon Hanson\",\"replies\":[]},{\"date\":\"1742177337.097879\",\"message\":\"IDB reference - <https://salesforce.quip.com/DSaJAnhkPVYa#temp:C:TRTe4d49a90afb84ae7bde9af390>\",\"username\":\"Jon Hanson\",\"replies\":[]}]},{\"date\":\"1742173862.060429\",\"message\":\"<@W012Z3N7ML7> were there any failover (or similar) activities for LA5LAPPIA41 cluster?  Oracle <https://la5-c1-ia4.salesforceliveagent.com/content/_internal/settings?format=simple&amp;match=oracle|settings> are as below:\\n```get.oracle.sessions.enabled=false\\noracle.dataguard.connect_timeout=10\\noracle.dataguard.enabled=false\\noracle.dataguard.failover=on\\noracle.dataguard.load_balance=off\\noracle.db=LA5DB1S\\noracle.ebr.enabled=true\\noracle.ebr.version=254\\noracle.host=<http://LA5DB1-ia4-scan.ops.sfdc.net|LA5DB1-ia4-scan.ops.sfdc.net>\\noracle.la.datamodel.pwd=_HiddenStringSettingEntry_ TuJck1sPaVxM7X3HVsOaImzmijQIitNFG435war45lE=\\noracle.la.datamodel.user=LADATAMODEL_LA\\noracle.la.package.name=liveagent\\noracle.logging=true\\noracle.logging.ignore=DoRoutingStatement, GetAllSessionsStatement, GetExpiredVisitorsStatement, GetCustomRoutingSyncRequestsStatement, QueryDual, GetActiveRacNode, GetDanglingCustomRoutingRequestsStatement, CheckCustomRoutingSynchronizedTimeoutStatement, QueryVersion\\noracle.monitoring.accumulatorSize=60\\noracle.pool.max=600\\noracle.pool.maxIdleTimeExcessConnectionsInSeconds=900\\noracle.pool.min=100\\noracle.port=5121\\noracle.pwd=_HiddenStringSettingEntry_ /lMSB61ewvgRIOqsI7gYrwFlz/m+RZ+X/b6lOPIEZ5I=\\noracle.ssl.enabled=true\\noracle.ssl.port=5122\\noracle.statement.in.batch.size=200\\noracle.statement.long.timeout.seconds=120\\noracle.statement.timeout.seconds=60\\noracle.sys.pwd=_HiddenStringSettingEntry_ uPrhQXVQ1f21uPgp7o1aZJ01mWSRogJPnrbd6Fg6XA0=\\noracle.use.cert.auth=false\\noracle.user=sneezy \\noracle.version=Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production\\nstate.oracle.enabled=true\\noracle.jdbc.timezoneAsRegion=false\\noracle.net.ssl_v
2025-03-17T10:04:02.118995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ersion=1.2```\\n\",\"username\":\"Guru Balse\",\"replies\":[{\"date\":\"1742174282.959669\",\"message\":\"<@WF8D3GCG7> Can you please provide me some specific time of spike?\",\"username\":\"Kapil Kothari\",\"replies\":[]},{\"date\":\"1742175390.614799\",\"message\":\"<@WF8D3GCG7> No DB failover since Feb 1st.\",\"username\":\"Kapil Kothari\",\"replies\":[]}]},{\"date\":\"1742174324.380529\",\"message\":\"<Vyshnavi Pinnu>\\n\\nhere is the heap dump\",\"username\":\"Sunny Chiu\",\"replies\":[],\"attachments\":[{\"name\":\"la5-lapp1-7-ia4.2025.03.16-20_33-UTC.logs.jar\",\"url\":\"https://files.slack.com/files-pri/T7T5PNK3P-F08HHMG6WLX/la5-lapp1-7-ia4.2025.03.16-20_33-utc.logs.jar\"}]},{\"date\":\"1742174750.297819\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742174782.520809\",\"message\":\"<Priyanka Sharma> this is the Gus case that requests an \\\"increase in Bulk API Batch Limit\\\" <https://gus.lightning.force.com/lightning/r/Case/500EE00001Tv6aZYAR/view>\",\"username\":\"Shahfazal Mohammed\",\"replies\":[{\"date\":\"1742174849.233969\",\"message\":\"<https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?earliest=-4h%40m&amp;latest=now&amp;q=search%20(index%3Ddistapps)%20logRecordType%3Dlasql%20(host%3D%22*la*%22)%20pod%3Dlapp5ia41%20%22GetServiceChannelsStatement%22%20%7C%20stats%20count%20by%20ORG_ID&amp;display.page.search.mode=smart&amp;dispatch.sample_ratio=1&amp;display.page.search.tab=statistics&amp;display.general.type=statistics&amp;display.statistics.sortColumn=count&amp;display.statistics.sortDirection=desc&amp;sid=1742174204.178546_09348F7E-5E76-4ECF-A2C1-286580FC4B6E|Splunk request> that shows the spike for this org (almost 290K in the last 4hrs) for the `GetServiceChannelsStatement` DB statement\",\"username\":\"Shahfazal Mohammed\",\"replies\":[]},{\"date\":\"1742175048.407489\",\"message\":\"<Priyanka Sharma> would be helpful if we could get someone to comment on this Case.\\n&gt; • *BlackTab Org Value Name:* Maximum number of Api Batch Items\\n&gt; • *Previous Org Value:* 15000\\n&gt; • *New Org Value:* 20000\\nand if it could cause a spike as well for this customer\",\"username\":\"Shahfazal Mohammed\",\"replies\":[]}]},{\"date\":\"1742175440.108149\",\"message\":\"Customer Case count :thread:\",\"username\":\"Neha Baghel\",\"replies\":[{\"date\":\"1742175457.273639\",\"message\":\"Customer case: 1\\nSocial mentions: 0\",\"username\":\"Neha Baghel\",\"replies\":[]},{\"date\":\"1742183671.519459\",\"message\":\"Customer case: 2\\nSocial mentions: 0\",\"username\":\"Neha Baghel\",\"replies\":[]},{\"date\":\"1742198534.603479\",\"message\":\"Customer case: 3\\nSocial mentions: 0\",\"username\":\"Neha Baghel\",\"replies\":[]},{\"date\":\"1742198544.429379\",\"message\":\"FYI <Nupoor Jain>\",\"username\":\"Neha Baghel\",\"replies\":[]}]},{\"date\":\"1742175565.374479\",\"message\":\"Chat rest calls count: <https://monitoring.internal.salesforce.com/argus/#/viewmetrics?expression=RATE(-1d:service-cloud-realtime.IA4.SP5.lapp*:app.platform.request.chat.rest.calls.Count%7Bdevice%3D*%7D:avg:1m-avg)|https://monitoring.internal.salesforce.com/argus/#/viewmetrics?expression=RATE(-1d:service-cloud-[…]hat.rest.calls.Count%7Bdevice%3D*%7D:avg:1m-avg)>\",\"username\":\"Guru Balse\",\"replies\":[],\"attachments\":[{\"name\":\"image.png\",\"url\":\"https://files.slack.com/files-pri/T7T5PNK3P-F08HXKZ4TV3/image.png\"}]},{\"date\":\"1742177135.065709\",\"message\":\"Hi Team, One of the sig customer <https://orgcs.lightning.force.com/lightning/r/Account/0013000001MYktjAAD/view|Engine (Hotel Engine)> who is on NA208 have also reported the issue with Omni Channel global case 470045129 Org 62 Case: <https://org62.lightning.force.com/500ed00000ENPFNAA5|45027998> customer is looking for an ETA is there any ETA can be shared with the customer? CC <Mallareddy Reddy>\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742177613.765149\",\"message\":\"Is hotel engine using live agent or they are using standard Omni with messaging for inapp and web \",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[{\"date\":\"1742177743.854229\",\"message\":\"<@WF8MT88EA> Thank you for responding, I checked the customer enhanced omni channel is disabled.\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742177771.381029\",\"message\":\"Are they using live agent? \",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742177940.177929\",\"message\":\"I checked the service channel and I see service channel for Voice, chat, standard messaging. However based on the information shared by the customer they are facing the issue with the voice calls.\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742177995.024219\",\"message\":\"*Agents Unable to Accept Calls*\\n• The \\\"Accept\\\" button is greyed out, preventing agents from answering calls.\\n• This has resulted in missed calls.\\n*Calls Stuck in Queue After Completion*\\n• Calls remain stuck in the *Member Support Call Queue* even after completion.\\n• Observed in the Omni Supervisor Queue Backlog, where completed calls continue to appear in the queue.\\n\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742179538.963309\",\"message\":\"<@WF8MT88EA> <Priyanka Sharma> Please confirm is there any update can be shared in Trust. The reason is customer is customer is looking for an ETA also they see that the trust post is not updated for an long time.\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742179654.438299\",\"message\":\"We have a solution now and it is under testing\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742180714.548699\",\"message\":\"<@WF8MT88EA> can we get the trust post updated? Because customer is looking for an update Customer Comment:\\nIt is unacceptable on our end to report this issue, not see the status page updated for hours and have no clue when it will be fixed. I appreciate your work on this Stanley and know you're working with the tools and information you have, but I need to communicate to the teams I am responsible for ensuring platform uptime to when this will be resolved.\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742180776.037769\",\"message\":\"<Jagruthi Akkaldevi> can you update the trust posting\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742180789.433469\",\"message\":\"Yup \",\"username\":\"Jagruthi Akkaldevi\",\"replies\":[]},{\"date\":\"1742180853.858329\",\"message\":\"thanks we should be out of it in max 30 min after the EVP approval ..an approx ETA should be 45 min ..but just update we have found the issue and running the deployment to fix it\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742180979.266639\",\"message\":\"Updated trust \",\"username\":\"Jagruthi Akkaldevi\",\"replies\":[]},{\"date\":\"1742186923.987249\",\"message\":\"<@WF8MT88EA> I have asked Hotel Engine to check if they can enable Enhanced Omni from messaging setting I am not seeing any standard messaging channel\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742187053.296609\",\"message\":\"Customer have enabled Enhanced Omni and doing the testing\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742187644.285779\",\"message\":\"Customer mentioned that so far it seems to be working.\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742187667.290609\",\"message\":\"Good sign \",\"username\":\"Jagruthi Akkaldevi\",\"replies\":[]},{\"date\":\"1742188476.982479\",\"message\":\"<Priyanka Sharma> <@WF8MT88EA> I had a discussion with Hotel Engine after enabling the omni the functionality is now working as expected they will use enhanced omni until the issue is fixed. The reason why they were using std omni is because the agents are not yet trained with the enhanced omni\",\"username\":\"Stanley K\",\"replies\":[]}]},{\"date\":\"1742177861.673679\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742177889.872259\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742179179.352129\",\"message\":\"<@WF8D3GCG7> please review the script <https://gus.lightning.force.com/lightning/r/Case/500EE00001Ty4M8YAJ/view>\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742179401.204579\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742179600.569269\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742179907.301159\",\"message\":\"<Priyanka Sharma> has joined acknowledging the page for *GUS: CC Edge Incident Escalation* service.\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742180762.382269\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742181433.918279\",\"message\":\"received verbal approval on call from <@WADPABC5P>\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742181806.997779\",\"message\":\"Check for all hosts:\\n```<https://la5-c1-ia4.salesforceliveagent.com/content/_internal/proxy/la5-lapp1-INDEX-ia4.ops.sfdc.net:8095/content/_internal/settings?format=simple&amp;match=core.instances>\\n-- 1\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 2\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 3\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 4\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 5\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 6\\nError 400 Bad Request\\nHTTP ERROR 400\\nP
2025-03-17T10:04:02.118995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] roblem accessing /content/_internal/proxy/la5-lapp1-6-ia4.ops.sfdc.net:8095/content/_internal/settings. Reason:\\n    Bad Request\\n-- 7\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443\\n-- 8\\ncore.instances=<http://na208.salesforce.com|na208.salesforce.com>\\ncore.instances.port=8443```\",\"username\":\"Guru Balse\",\"replies\":[]},{\"date\":\"1742181817.806829\",\"message\":\"<@WAQM68L1W> ^^\",\"username\":\"Guru Balse\",\"replies\":[]},{\"date\":\"1742183646.115449\",\"message\":\"<https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=1742176648.119599_4A14D634-95F4-4C30-A7CA-5B6D5306162A&amp;dispatch.sample_ratio=1&amp;display.events.fields=%5B%22host%22%2C%22source%22%2C%22sourcetype%22%2C%22falcon_instance%22%2C%22cell%22%2C%22k8s_pod_name%22%2C%22agenttime%22%5D&amp;display.page.search.mode=fast&amp;display.visualizations.charting.chart=column&amp;q=search%20index%3Ddistapps%20AND%20sourcetype%3Dliveagentserver1%3A*%20AND%20%20datacenter%3Dia4%20%20pod%3Dlapp5ia41%20%7Csearch%20%22Error%20validating%20the%20appUrls%20for%20core%20instance%22%20AND%20NOT%20%22na207%22&amp;earliest=-7d%40h&amp;latest=now|https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?sid=174[…]7%22&amp;earliest=-7d%40h&amp;latest=now>\",\"username\":\"Bhasker Konakanchi\",\"replies\":[]},{\"date\":\"1742183651.289419\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742185578.120649\",\"message\":\"<Jagruthi Akkaldevi> I believe the team is working to fix the issue can we please update the trust if the issue is still not fixed?\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742185774.300919\",\"message\":\"Here is the 1-2 jar for heap/thread dump analysis - this is post iDB fixes\",\"username\":\"Guru Balse\",\"replies\":[{\"date\":\"1742188476.677799\",\"message\":\"<@WF8D3GCG7>: Does it take time for the fixes to be effective?\",\"username\":\"Vyshnavi Pinnu\",\"replies\":[]},{\"date\":\"1742189902.674409\",\"message\":\"I see about same number of threads in \\\"Timed_waiting\\\" state on \\\"REST|GET|Availability|\\\" before and after the iDB fixes\",\"username\":\"Vyshnavi Pinnu\",\"replies\":[]},{\"date\":\"1742189972.913139\",\"message\":\"REST|GET|Availability| threads in \\\" Timed_waiting\\\" before : 1842\\nREST|GET|Availability| threads in \\\" Timed_waiting\\\" After fix : 1826\",\"username\":\"Vyshnavi Pinnu\",\"replies\":[]}],\"attachments\":[{\"name\":\"la5-lapp1-2-ia4.2025.03.17-04_19-UTC.logs.jar\",\"url\":\"https://files.slack.com/files-pri/T7T5PNK3P-F08J3ADJ44U/la5-lapp1-2-ia4.2025.03.17-04_19-utc.logs.jar\"}]},{\"date\":\"1742186342.625569\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742188468.422189\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742190949.948239\",\"message\":\"<@WAS176048> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742191083.794169\",\"message\":\"Tuan Nguyen has paged 1 team.\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742191084.686259\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742191978.386759\",\"message\":\"Tuan Nguyen has paged 1 team.\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742191979.183059\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742192231.415049\",\"message\":\"Andy Lintner\\n2:16 PM\\nWe want to block all requests to the off-core lapp server in IA4 that start with \\\"<https://d.la5-c1-ia4.salesforceliveagent.com/chat/rest/Visitor/Availability.jsonp?org_id=00DA0000000abSm>\\\"\\n\\nA customer's broken implementation is causing a flood of requests post instance refresh. This is impacting all customers on this lapp instance. We need to block these requests to the lapp servers (not Core). The lapp servers sit behind the F5 load balancer, but are not managed via BlackTab\",\"username\":\"Tuan Nguyen\",\"replies\":[]},{\"date\":\"1742192252.024799\",\"message\":\"<@W01284YJ15F> ^^ action needed here\",\"username\":\"Tuan Nguyen\",\"replies\":[]},{\"date\":\"1742192525.230859\",\"message\":\"<Bhoomika B A> <Omini Rajguru> GUS team is requesting to raise outbound case for <https://orgcs.lightning.force.com/lightning/r/Account/0013000000ekCLWAA2/view|Schneider Electric USA, Inc.> Since their custom implementation is causing the issue. Customer org id is <https://d.la5-c1-ia4.salesforceliveagent.com/chat/rest/Visitor/Availability.jsonp?org_id=00DA0000000abSm|00DA0000000abSm> can you help in raising the outbound case cc <Md Rahman Khan> <Mallareddy Reddy>\",\"username\":\"Stanley K\",\"replies\":[{\"date\":\"1742192557.344429\",\"message\":\"FYI <Ragul Vasu>\",\"username\":\"Bhoomika B A\",\"replies\":[]},{\"date\":\"1742192939.562159\",\"message\":\"Hi <Stanley K> Can we have more information on what should be communicated to the customer? This will help in creating the case with the correct context\",\"username\":\"Ragul Vasu\",\"replies\":[]},{\"date\":\"1742193105.887149\",\"message\":\"As per the discussion customer org instance has been refreshed during weekend from na206 to na253 once after the instance refresh. Customer custom implementation is actually causing the traffic for the other customers so this change has to disabled and it has to be communicated with the customer FYI <@WF8MT88EA> <Jagruthi Akkaldevi> please do share more information if I am missing something\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742193118.052199\",\"message\":\"CC <Priyanka Sharma>\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742193319.078029\",\"message\":\"Hi FOlks\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742193457.832929\",\"message\":\"Schneider's broken implementation of chat is causing a flood of requests post instance refresh due to a hardcoded URL. This is impacting other customers on this live agent instance.\\n\\nSchneider's chat is probably not working, Schneider's broken implementation of chat is causing a flood of requests post instance refresh. This is impacting all customers on this live agent instance. We need to block these requests to the live agent app servers (not Core).\\n\\nScheider's chat is probably not working. The fastest way they need to fix is to replace live agent instance with Schneider Electric needs to go to <http://la4-c2-ia7.salesforceliveagent.com|la4-c2-ia7.salesforceliveagent.com> instead of <http://la5-c1-ia4.salesforceliveagent.com|la5-c1-ia4.salesforceliveagent.com>\\n\\nWe can talk to the customer if they need help.the fastest way to fix, is to replace live agent instance with Schneider Electric needs to go to <http://la4-c2-ia7.salesforceliveagent.com|la4-c2-ia7.salesforceliveagent.com> instead of <http://la5-c1-ia4.salesforceliveagent.com|la5-c1-ia4.salesforceliveagent.com> Thanks\\n\\nWe can talk to the customer if they need help.\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742193491.291209\",\"message\":\"Ack! Creating the case. Will share the case details shortly.\",\"username\":\"Ragul Vasu\",\"replies\":[]},{\"date\":\"1742193517.825839\",\"message\":\"Could you please get me the CSM and AE's phone numbers?\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742193531.360259\",\"message\":\"<@WF8MT88EA> CSM: *Abinaya Thedsanamoorthy*\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742193620.421559\",\"message\":\"Great\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742193641.748429\",\"message\":\"<Abinaya Thedsanamoorthy> Good morning\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742194072.597969\",\"message\":\"<@WF8MT88EA> Can you please confirm the severity of the outbound case? Should it be made a Sev-1? or a Sev-2 will suffice?\",\"username\":\"Ragul Vasu\",\"replies\":[]},{\"date\":\"1742194224.390839\",\"message\":\"It would be Sev1\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742194245.387549\",\"message\":\"They can call me at 4155307600\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742194262.099229\",\"message\":\"Ack! Please find Case # <https://orgcs.lightning.force.com/lightning/r/Case/500Hx00000luFsXIAU/view|470047199>\",\"username\":\"Ragul Vasu\",\"replies\":[]},{\"date\":\"1742195497.336469\",\"message\":\"<@WF8MT88EA> Called the customer informed customer about the issue and ask them to join the meeting to discuss further\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742195510.714289\",\"message\":\"<https://salesforce.webex.com/salesforce/j.php?MTID=mfe4dceb93625a7820bc359e8f9769509>\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742195584.930349\",\"message\":\"<@WF8MT88EA> please confirm if you will be joining the meeting\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742195774.467499\",\"message\":\"Sure. Will have others also to join\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]}]},{\"date\":\"1742192838.704679\",\"message\":\"<@WAS176048> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742192996.630359\",\"message\":\"block uri -- \\\"<https://d.la5-c1-ia4.salesforceliveagent.com/chat/rest/Visitor/Availability.jsonp?org_id=00DA0000000abSm>\\\"\",\"username\":\"Tuan Nguyen\",\"replies\":[]},{\"date\":\"1742194906.519199\",\"message\":\"\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742195303.221489\",\"message\":\"<Priyanka Sharma> has joined acknowledging the page for *Global Netw
2025-03-17T10:04:02.118995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] ork Operations (GNO)* service.\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742195424.686709\",\"message\":\"<Priyanka Sharma> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742195753.592989\",\"message\":\"2025-03-17 07:15:53 UTC - @SRE : Please help to approve 73103005 for Emergency Failover (NA208-ia5) (ops0-release2-1-ia5)\",\"username\":\"user not found\",\"replies\":[]},{\"date\":\"1742196593.708589\",\"message\":\"<@WF8MT88EA> / <Bhasker Konakanchi> can you please fill in with what you are updating to customer so that we can update the trust accordingly\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742197014.192749\",\"message\":\"A few developers joined the call and they are in the process of changing the URL\",\"username\":\"Sudhakara Reddy Peddi\",\"replies\":[]},{\"date\":\"1742197531.935569\",\"message\":\"<@WF8MT88EA> <Abinaya Thedsanamoorthy> <https://www.se.com/ww/en/> when I check this URL page source I am still seeing <https://c.la5-c1-ia4.salesforceliveagent.com> not the recent <http://la4-c2-ia7.salesforceliveagent.com|la4-c2-ia7.salesforceliveagent.com> I believe as they mentioned it doesn't seems to be refreshed\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742197583.788689\",\"message\":\"<Jagruthi Akkaldevi> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742198198.744659\",\"message\":\"Splunk query showing unique query params fetched by Schneider per minute. We want to see this drop: <https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?dispatch.sample_ratio=1&amp;display.events.fields=%5B%22host%22%2C%22source%22%2C%22sourcetype%22%2C%22falcon_instance%22%2C%22cell%22%2C%22k8s_pod_name%22%2C%22agenttime%22%5D&amp;display.page.search.mode=fast&amp;display.visualizations.charting.chart=line&amp;q=search%20index%3Ddistapps%20AND%20sourcetype%3Dliveagentserver1%3A*%20AND%20%20datacenter%3Dia4%20%20pod%3Dlapp5ia41%20availability%2000DA0000000abSm%20earliest%3D-1h%40m%20latest%3D-5m%40m%20%7C%20timechart%20span%3D1m%20dc(requestUri)&amp;earliest=-15m&amp;latest=now&amp;display.general.type=visualizations&amp;display.page.search.tab=visualizations&amp;sid=1742198104.232882_B07B4BA0-F02B-4D7E-9890-2524097BB5BC|https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?dispatc[…]4BA0-F02B-4D7E-9890-2524097BB5BC>\",\"username\":\"Andy Lintner\",\"replies\":[]},{\"date\":\"1742198517.152839\",\"message\":\"CIC IC has been handed over from <Neha Baghel> to <Nupoor Jain> :thankyou_:\",\"username\":\"Neha Baghel\",\"replies\":[]},{\"date\":\"1742198651.538009\",\"message\":\"<@WV8TM5WP5>\",\"username\":\"Priyanka Sharma\",\"replies\":[]},{\"date\":\"1742198697.243559\",\"message\":\"PRB to SCRT1\",\"username\":\"Krisztian Zsibrek\",\"replies\":[{\"date\":\"1742198924.759959\",\"message\":\"SCRT1 DevOps is the right team.\",\"username\":\"Narendra Singh Shekhawat\",\"replies\":[]}]},{\"date\":\"1742198933.880369\",\"message\":\"currently waiting for customer's config to propagate\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742198949.654499\",\"message\":\"we are monitoring the change\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742199692.210309\",\"message\":\"when customer confirms the changes, SCRT team is going to restart their service\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200211.853279\",\"message\":\"from the EMEA site, the URL for schneider changed to ia7\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200217.315969\",\"message\":\"CUJ is still red\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200407.734589\",\"message\":\"splunk numbers are a bit lower now\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200424.530129\",\"message\":\"we see three greens out of five for CUJ (could be misleading)\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200750.444849\",\"message\":\"scrt metrics: <https://moncloud-grafana.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl/d/haPuS7gnz/scrt1-1p-metrics-scrt-devops?orgId=1&amp;var-DataCenter=IA4&amp;var-SuperPod=SP5&amp;var-Cluster=lapp5ia41&amp;var-Host=All&amp;var-Scrtkafka_Cluster=scrtkafka5ia41&amp;var-Scrtkafka_Hosts=All&amp;from=now-3h&amp;to=now-5m&amp;viewPanel=172&amp;refresh=1m|https://moncloud-grafana.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl/d/haPuS7gnz/scrt1-1p-metrics-scrt-d[…]h&amp;to=now-5m&amp;viewPanel=172&amp;refresh=1m>\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200804.683529\",\"message\":\"site switch would be detrimental for customers with hard coded URL's\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200882.457449\",\"message\":\"currently waiting for customer confirmation for CDN refresh, then SCRT will restart the service\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742200921.719799\",\"message\":\"Splunk query with Schneider requests: <https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?dispatch.sample_ratio=1&amp;display.events.fields=%5B%22host%22%2C%22source%22%2C%22sourcetype%22%2C%22falcon_instance%22%2C%22cell%22%2C%22k8s_pod_name%22%2C%22agenttime%22%5D&amp;display.page.search.mode=fast&amp;display.visualizations.charting.chart=line&amp;q=search%20index%3Ddistapps%20AND%20sourcetype%3Dliveagentserver1%3A*%20AND%20%20datacenter%3Dia4%20%20pod%3Dlapp5ia41%20availability%2000DA0000000abSm&amp;earliest=-60m%40m&amp;latest=now&amp;sid=1742200674.19295_B812511E-980F-4AB8-AA54-CED8851C5C99|https://splunk-web-noncore.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/search/search?dispatc[…]511E-980F-4AB8-AA54-CED8851C5C99>\",\"username\":\"Bhasker Konakanchi\",\"replies\":[]},{\"date\":\"1742201337.980199\",\"message\":\"customer is doing one country by one country cache refresh\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742201343.319149\",\"message\":\"new ETA 20-25 minutes\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742201453.211819\",\"message\":\"<@WV8TM5WP5> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742202193.807349\",\"message\":\"FYI <Gaurang Joshi>\",\"username\":\"Stanley K\",\"replies\":[]},{\"date\":\"1742202265.579819\",\"message\":\"we are proceeding with site switch\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742202344.744089\",\"message\":\"trust posting finished\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742202377.146999\",\"message\":\"starting site switch\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742202404.654689\",\"message\":\"ran by <Weiling Hu>\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742202415.672499\",\"message\":\"2025-03-17 09:06:55 UTC - Start of Site Switch  (Renderer <http://weiling.hu|weiling.hu>) (NA208-ia5) (ops0-release2-1-ia5)\",\"username\":\"user not found\",\"replies\":[]},{\"date\":\"1742202420.244219\",\"message\":\"*message to post after the site switch -* <https://help.salesforce.com/s/articleView?id=000382130&amp;type=1>\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742202526.065429\",\"message\":\"<@WV8TM5WP5> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742202821.344449\",\"message\":\"2025-03-17 09:13:40 UTC - Database is now ready at New Primary, starting app servers (NA208-ia5) (ops0-release2-1-ia5)\",\"username\":\"user not found\",\"replies\":[]},{\"date\":\"1742203236.127439\",\"message\":\"2025-03-17 09:20:35 UTC - End of Service Disruption for Site Switch (Instance is now ALIVE), performing kaiju tests (NA208-ia5) (ops0-release2-1-ia5)\",\"username\":\"user not found\",\"replies\":[]},{\"date\":\"1742203313.200439\",\"message\":\"2025-03-17 09:21:51 UTC - End of Site Switch, Kaiju Test completed. App Tier Rolling Restart still ongoing. (NA208-ia5) (ops0-release2-2-ia5)\\n<https://splunk-web.log-analytics.monitoring.aws-esvc1-useast2.aws.sfdc.is/en-US/app/publicSharing/search?q=search%20(index%3DNA208)%20%60logRecordType(L)%60%20starttime%3D03%2F17%2F2025%3A08%3A21%3A00%20endtime%3D03%2F17%2F2025%3A09%3A31%3A00%20%7C%20eval%20result%3Dif(resultCode%3D0%2C0%2C1)%20%7C%20eval%20combi%3D(isReadonly%20%2B%20%22%40%22%20%2B%20%20datacenter%20%2B%20%22%40%22%20%2B%20result)%20%7C%20timechart%20span%3D1m%20count%20by%20combi%20%7Crename%200%40ia4%400%20AS%20ReadWrite%40IA4_success%200%40ia5%400%20AS%20ReadWrite%40IA5_success%201%40ia4%400%20AS%20ReadOnly%40IA4_success%201%40ia5%400%20AS%20ReadOnly%40IA5_success%200%40ia4%401%20AS%20ReadWrite%40IA4_failed%200%40ia5%401%20AS%20ReadWrite%40IA5_failed%201%40ia4%401%20AS%20ReadOnly%40IA4_failed%201%40ia5%401%20AS%20ReadOnly%40IA5_failed&amp;display.page.search.tab=visualizations&amp;display.general.type=visualizations|View Readwrite activities in Splunk>\",\"username\":\"user not found\",\"replies\":[]},{\"date\":\"1742203445.207249\",\"message\":\"app restart is done, SCRT team is checking their service\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742203622.588589\",\"message\":\"<@WV8TM5WP5> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742203920.065629\",\"message\":\"Liveagent is now healthy\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742203931.448669\",\"message\":\"new liveagent cluster is <http://la5-c1-ia5.salesforceliveagent.com|la5-c1-ia5.salesforceliveagent.com>\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204059.654089\",\"message\":\"Weiling Hu\\n10:32\\n@Terry He\\nas we need to do code checki
2025-03-17T10:04:02.118995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] n immediately after siteswitch, can you help to ask the EVP to chatter approval to the ERR too?\\n<https://gus.lightning.force.com/lightning/r/eRelease_Request__c/a16EE0000005odhYAA/view>\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204067.826099\",\"message\":\"we are monitoring metrics\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204100.026629\",\"message\":\"customers using hard coded liveagent URL's need to change theirs, using this help trust posting\\n\\n<https://help.salesforce.com/s/articleView?id=000382130&amp;type=1>\\n\\n\\nPlease note that site switch will result in a change in Salesforce Chat Endpoint URLs. Customers with hard-coded references to their current Endpoint URL will need to update these references once the maintenance is complete to ensure Salesforce Chat functionality continues to work properly.  We advise customers to review the following article and take the necessary steps to prevent any disruption to chat functionality -\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204285.120679\",\"message\":\"support is reaching out to customer\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204299.691159\",\"message\":\"swimlanes:\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204327.736939\",\"message\":\"1. CSG checking with customers with cases\\n2. Site Switch team doing post tasks\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742204363.445479\",\"message\":\"Pavani Parimi\\n10:38\\nHi Team, I am from the Support team taking over a Premier sev-1 case during EMEA hours (B &amp; H Photo &amp; Electronics)\\n\\nI have checked with updated Live Agent end point in Postman and the response is quicker. Omni Supervisor also loads better.\\n\\nI am checking with the customer to validate.\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742205076.060889\",\"message\":\"B&amp;H - chat widget is loading fine and I see them handling switch server fine\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742205340.595559\",\"message\":\"end time: 9:30 UTC\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742205440.187699\",\"message\":\"<@WV8TM5WP5> resolved the incident. Would you also like to send a TCE?\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742205508.916679\",\"message\":\"<@WV8TM5WP5> sent a new internal comm:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742205523.002979\",\"message\":\"<AIOpsAgent> generate PRB for this case\",\"username\":\"Krisztian Zsibrek\",\"replies\":[{\"date\":\"1742205525.795609\",\"message\":\":information-blue: _I'm processing your request now. You can follow my progress <https://sherlock-rcaapi.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl:7443/smartui/thread/1742205523.002979/message/0|here>._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205536.204199\",\"message\":\"_Created action plan..._ View button\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205538.308439\",\"message\":\":aiops-agent: Could you please provide the case ID for the incident you want detailed information on?\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205564.077539\",\"message\":\"73095026\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]},{\"date\":\"1742205605.864079\",\"message\":\"<@WU94VGVQE> FYI, a bit of a bug here\",\"username\":\"Krisztian Zsibrek\",\"replies\":[]}]},{\"date\":\"1742205622.300919\",\"message\":\"<AIOpsAgent> generate PRB for 73095026\",\"username\":\"Krisztian Zsibrek\",\"replies\":[{\"date\":\"1742205624.531719\",\"message\":\":information-blue: _I'm processing your request now. You can follow my progress <https://sherlock-rcaapi.sfproxy.monitoring.aws-esvc1-useast2.aws.sfdc.cl:7443/smartui/thread/1742205622.300919/message/0|here>._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205637.500319\",\"message\":\"_Created action plan..._ View button\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205639.205029\",\"message\":\":workflow_ticket_closed: _Running fetch_incident_details_for_case_id to fetch detailed incident information from GUS for case ID 73095026...._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205652.509169\",\"message\":\":workflow_ticket_closed: _Running fetch_tce_details_for_case_id to fetch TCE details for case ID 73095026...._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205665.495049\",\"message\":\":workflow_ticket_closed: _Running fetch_ica_summary_for_case_id to retrieve ICA summary for case ID 73095026...._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205691.831119\",\"message\":\":workflow_ticket_closed: _Running prb_generation_tool to generate PRB document using all gathered information for case ID 73095026...._\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205704.072039\",\"message\":\"*Title:*\\n Reduced Core App Successful Logins in Pod na208\\n\\n*Summary:*\\n On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod `na208` was detected. The incident commenced at approximately `09:07 UTC` and persisted until `09:11 UTC`, with successful logins plummeting from the threshold of `30` to `0`. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\\n\\n*Immediate Resolution:*\\n An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance. However, no immediate resolution was documented for the reduced login success rate, indicating the need for manual intervention and further investigation.\\n\\n*Proximate Cause:*\\n The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the LAP servers. The cause of the reduced Core App login success rate remains undetermined, with no causal metrics identified and potential data analysis gaps noted.\\n\\n*After Action Questions:*\\n What improvements can be made to the data analysis process to ensure more accurate causation identification?\\nHow can we better monitor and alert for certificate-related issues to prevent similar incidents in the future?\\nWhat training or documentation updates are needed to ensure teams can swiftly identify and respond to similar incidents?\\n\\n*Data sources:*\\n ICA\\nOther sources\\nCase\\n\\n*Suggestions:*\\n Do you have any suggestions or improvements to make the generated summary more clear, concise, or accurate?\",\"username\":\"AIOpsAgent\",\"replies\":[]},{\"date\":\"1742205714.776389\",\"message\":\"AIOps Agent can make mistakes. Your feedback helps improve it. :thumbsup: button :thumbsdown: button\",\"username\":\"AIOpsAgent\",\"replies\":[]}]},{\"date\":\"1742205742.094719\",\"message\":\":alarm-clock2: The below reminder is NOW DUE:\",\"username\":\"Incident Command Center\",\"replies\":[]},{\"date\":\"1742205838.822599\",\"message\":\"ICC is in the final stages of handling the incident - just a little more patience, and we’ll be good to go! :smile::+1:\\nIn the meantime, here are the closing details:\",\"username\":\"Incident Command Center\",\"replies\":[]}]"}
2025-03-17T10:04:04.231834+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844231,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages remove redis keys for docId QWCAAANuCF5"}
2025-03-17T10:04:04.245010+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844244,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages produceEvent for docId QWCAAANuCF5"}
2025-03-17T10:04:04.245012+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844244,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"produceEvent with link https://salesforce.quip.com/M8S4Ap02KbIu for event 13054fa1-b6e2-45d6-8afe-0c7523384b29 and case 73095026"}
2025-03-17T10:04:04.245133+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricRequestExecute] - icc-report-finished - undefined"}
2025-03-17T10:04:04.245178+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.245250+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.245421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:04.245619+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key '13054fa1-b6e2-45d6-8afe-0c7523384b29' to topic 'incident': {\"createdAt\":\"2025-03-17T10:04:04.245Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/M8S4Ap02KbIu\"}}"}
2025-03-17T10:04:04.245733+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key '13054fa1-b6e2-45d6-8afe-0c7523384b29' to Kafka topic 'incident'"}
2025-03-17T10:04:04.245779+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricDurationTime] - icc-report-finished - undefined"}
2025-03-17T10:04:04.245830+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.30186500400304794\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.245906+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.30186500400304794\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.245963+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:04.246191+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844246,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-incident-closed correctly routed"}
2025-03-17T10:04:04.694032+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844693,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:04.694327+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-report-finished' for consumer group='icc-reporting'"}
2025-03-17T10:04:04.694414+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:04:04.245Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/M8S4Ap02KbIu\"}}"}
2025-03-17T10:04:04.694417+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.694449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:04:04.694582+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:04.694583+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-report-finished"}
2025-03-17T10:04:04.694586+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-report-finished is now being processed."}
2025-03-17T10:04:04.694770+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T10:04:04.694844+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:04.694983+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T10:04:04.694984+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:04:04.306Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"\"},\"caseRecordId\":\"500EE00001Ty0FFYAZ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:04:04.306Z\",\"recordId\":\"500EE00001Ty0FFYAZ\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseNumber\":\"73095026\"},\"changes\":{}}"}
2025-03-17T10:04:04.694985+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:04:04.694986+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844694,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:04:04.695061+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:04.695061+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-gus-response"}
2025-03-17T10:04:04.695064+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T10:04:04.695093+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T10:04:04.695500+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,45,102,105,110,105,115,104,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,48,52,58,48,52,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,105,110,103]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:04:04.245Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J0HPF5H9\\\",\\\"name\\\":\\\"icc-73095026\\\"},\\\"subject\\\":\\\"Customers seeing latency on OMNI chat - single pod\\\",\\\"description\\\":\\\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\\\",\\\"detectedTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Closed\\\",\\\"origin\\\":\\\"Customer - External\\\",\\\"originReference\\\":\\\"--\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T00:26:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T09:30:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hbh-zkju-hsu\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\\\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Configuration\\\",\\\"createdAt\\\":\\\"2025-03-17T00:26:10.551Z\\\",\\\"closingDetails\\\":{\\\"productTag\\\":\\\"a1aEE000000RZw9YAG\\\",\\\"detectedByServiceOwner\\\":true,\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"proximateCause\\\":\\\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\\\",\\\"customerExperience\\\":\\\"Chat function not working\\\",\\\"immediateResolution\\\":\\\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\\\",\\\"aarQuestions\\\":\\\"- Why were we so slow to implement the solution?\\\",\\\"changes\\\":\\\"\\\",\\\"breakFix\\\":\\\"\\\",\\\"productTagName\\\":\\\"SCRT1 Central\\\",\\\"closedBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"incidentId\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"firstName\\\":\\\"Krisztian\\\",\\\"lastName\\\":\\\"Zsibrek\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"createdBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"modifiedBy\\\":\\\"kzsibrek@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-07-15T09:24:15.476Z\\\",\\\"modifiedAt\\\":\\\"2022-07-15T09:24:15.476Z\\\"},\\\"closingComments\\\":null,\\\"createdAt\\\":\\\"2025-03-17T10:03:58.286Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T10:03:58.286Z\\\",\\\"id\\\":\\\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\\\"},\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73095026\\\",\\\"caseRecordId\\\":\\\"500EE00001Ty0FFYAZ\\\",\\\"servicesImpacted\\\":[\\\"Core 1P\\\"],\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/M8S4Ap02KbIu\\\",\\\"engagedAt\\\":\\\"2025-03-17T00:26:23.518Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T09:57:25.281Z\\\"},\\\"eventObj\\\":{\\\"link\\\":\\\"https://salesforce.quip.com/M8S4Ap02KbIu\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:04:04.245Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-fee
2025-03-17T10:04:04.695500+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] f-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/M8S4Ap02KbIu\"}},\"encoder\":\"json\",\"eventType\":\"icc-report-finished\",\"payload\":{\"createdAt\":\"2025-03-17T10:04:04.245Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"icc-73095026\"},\"subject\":\"Customers seeing latency on OMNI chat - single pod\",\"description\":\"On March 17, 2025, an incident involving a drastic drop in successful logins for the Core App within pod na208 was detected. The incident commenced at approximately 09:07 UTC and persisted until 09:11 UTC, with successful logins plummeting from the threshold of 30 to 0. Concurrently, a separate issue was logged under GUS Case ID 73095026, involving latency on the OMNI chat service within the same pod, lasting 544 minutes and impacting customer performance. This was marked as resolved by executing an emergency site switch, addressing certificate errors and a broken implementation by Schneider Electric, which caused a flood of requests. Despite the resolution of the OMNI chat issue, the root cause of the login failure remains unidentified, necessitating further investigation.\",\"detectedTime\":\"2025-03-17T00:26:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Customer - External\",\"originReference\":\"--\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T00:26:00.000Z\",\"impactEndTime\":\"2025-03-17T09:30:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hbh-zkju-hsu\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-207-613-8053, pin: 347526838; \\nFind numbers in more regions on https://tel.meet/hbh-zkju-hsu?pin=7765220428500, pin: 7765220428500\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Configuration\",\"createdAt\":\"2025-03-17T00:26:10.551Z\",\"closingDetails\":{\"productTag\":\"a1aEE000000RZw9YAG\",\"detectedByServiceOwner\":true,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The proximate cause of the OMNI chat latency was identified as certificate errors and a broken implementation by Schneider Electric, leading to excessive requests to the Live Agent servers.\",\"customerExperience\":\"Chat function not working\",\"immediateResolution\":\"An emergency site switch was executed to mitigate the impact of the OMNI chat latency issue, which led to improved performance.\",\"aarQuestions\":\"- Why were we so slow to implement the solution?\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"SCRT1 Central\",\"closedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"createdBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"modifiedBy\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"email\":\"kzsibrek@salesforce.com\",\"active\":true,\"name\":\"Krisztian Zsibrek\",\"firstName\":\"Krisztian\",\"lastName\":\"Zsibrek\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"createdBy\":\"kzsibrek@salesforce.com\",\"modifiedBy\":\"kzsibrek@salesforce.com\",\"createdAt\":\"2022-07-15T09:24:15.476Z\",\"modifiedAt\":\"2022-07-15T09:24:15.476Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:03:58.286Z\",\"modifiedAt\":\"2025-03-17T10:03:58.286Z\",\"id\":\"fd63de7d-feef-4d3f-add3-62f54f8e0dba\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73095026\",\"caseRecordId\":\"500EE00001Ty0FFYAZ\",\"servicesImpacted\":[\"Core 1P\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/M8S4Ap02KbIu\",\"engagedAt\":\"2025-03-17T00:26:23.518Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T09:57:25.281Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/M8S4Ap02KbIu\"}}}"}
2025-03-17T10:04:04.695500+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:04.695501+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-report-finished does not have the necessary information to create a document"}
2025-03-17T10:04:04.695504+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-report-finished correctly routed"}
2025-03-17T10:04:04.695611+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,48,52,58,48,52,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:04:04.306Z\\\",\\\"user\\\":{\\\"id\\\":\\\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\\\",\\\"gusUserId\\\":\\\"005B0000006BPsrIAG\\\",\\\"slackUserId\\\":\\\"WV8TM5WP5\\\",\\\"name\\\":\\\"Krisztian Zsibrek\\\",\\\"email\\\":\\\"kzsibrek@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08J0HPF5H9\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001Ty0FFYAZ\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_closed\\\",\\\"timestamp\\\":\\\"2025-03-17T10:04:04.306Z\\\",\\\"recordId\\\":\\\"500EE00001Ty0FFYAZ\\\",\\\"incidentId\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"caseNumber\\\":\\\"73095026\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:04:04.306Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"\"},\"caseRecordId\":\"500EE00001Ty0FFYAZ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:04:04.306Z\",\"recordId\":\"500EE00001Ty0FFYAZ\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseNumber\":\"73095026\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T10:04:04.306Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"\"},\"caseRecordId\":\"500EE00001Ty0FFYAZ\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:04:04.306Z\",\"recordId\":\"500EE00001Ty0FFYAZ\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseNumber\":\"73095026\"},\"changes\":{}}}"}
2025-03-17T10:04:04.695612+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:04.695612+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-gus-response does not have the necessary information to create a document"}
2025-03-17T10:04:04.695613+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205844695,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-gus-response correctly routed"}
2025-03-17T10:04:09.730928+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205849729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:09.730934+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205849729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-prb-number-updated' for consumer group='icc-reporting'"}
2025-03-17T10:04:09.730935+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205849729,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:04:09.676Z\",\"user\":{\"id\":\"182908c0-0f8e-4f77-8ff9-b1a7435b6deb\",\"gusUserId\":\"005B0000006BPsrIAG\",\"slackUserId\":\"WV8TM5WP5\",\"name\":\"Krisztian Zsibrek\",\"email\":\"kzsibrek@salesforce.com\"},\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"C08J0HPF5H9\",\"name\":\"\"},\"caseRecordId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"prbNumber\":{\"sfid\":\"a4QEE000000684X2AQ\",\"valid\":true,\"gusIdentifier\":\"PRB-0027123\",\"value\":\"PRB-0027123\",\"createdByIcc\":true,\"workItems\":[{\"id\":\"a07EE00002BAtjwYAD\",\"success\":true}]}},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:04:09.676Z\",\"recordId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"caseNumber\":\"\"}}"}
2025-03-17T10:04:09.730936+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-prb-number-updated\"}}"}
2025-03-17T10:04:09.730937+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-prb-number-updated\"}}"}
2025-03-17T10:04:09.730938+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:09.730938+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-prb-number-updated"}
2025-03-17T10:04:09.730938+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-prb-number-updated is not consumed by icc-reporting."}
2025-03-17T10:04:09.730939+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205849730,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-prb-number-updated correctly routed"}
2025-03-17T10:04:10.757400+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205850757,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885251072,\"uptime\":2025746.14,\"totalMem\":4018151424,\"loadavg\":[0.05,0.05,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24284860,\"nice\":16990,\"sys\":3487030,\"idle\":1990470900,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24000420,\"nice\":16540,\"sys\":3503680,\"idle\":1990696380,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":30021,\"num\":2166,\"sum\":59914}}}"}
2025-03-17T10:04:10.757689+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205850757,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:14.457555+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:04:14.457642+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-ctc-lock-response' for consumer group='icc-reporting'"}
2025-03-17T10:04:14.457644+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"locks\":[],\"incidentId\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\"}"}
2025-03-17T10:04:14.457648+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T10:04:14.457686+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T10:04:14.457769+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:04:14.458081+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742205854457,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:05:10.765418+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742205910765,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884984832,\"uptime\":2025806.15,\"totalMem\":4018151424,\"loadavg\":[0.02,0.03,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24285480,\"nice\":16990,\"sys\":3487110,\"idle\":1990530150,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24001130,\"nice\":16540,\"sys\":3503770,\"idle\":1990755570,\"irq\":0}}],\"eventLoopStats\":{\"min\":131,\"max\":29853,\"num\":5,\"sum\":60004}}}"}
2025-03-17T10:05:10.765468+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742205910765,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:08:10.837402+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206090837,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884993024,\"uptime\":2025986.22,\"totalMem\":4018151424,\"loadavg\":[0.08,0.03,0.01],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24286650,\"nice\":16990,\"sys\":3487360,\"idle\":1990707890,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24002700,\"nice\":16540,\"sys\":3504020,\"idle\":1990933410,\"irq\":0}}],\"eventLoopStats\":{\"min\":159,\"max\":30027,\"num\":3,\"sum\":60027}}}"}
2025-03-17T10:08:10.837472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206090837,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:11:10.901475+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206270901,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2885021696,\"uptime\":2026166.29,\"totalMem\":4018151424,\"loadavg\":[0.02,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24287650,\"nice\":16990,\"sys\":3487650,\"idle\":1990885580,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24003690,\"nice\":16540,\"sys\":3504240,\"idle\":1991111200,\"irq\":0}}],\"eventLoopStats\":{\"min\":180,\"max\":30030,\"num\":3,\"sum\":60029}}}"}
2025-03-17T10:11:10.901516+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206270901,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:18:11.045407+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206691045,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2884550656,\"uptime\":2026586.43,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24291430,\"nice\":16990,\"sys\":3488240,\"idle\":1991300210,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24006930,\"nice\":16540,\"sys\":3504870,\"idle\":1991526100,\"irq\":0}}],\"eventLoopStats\":{\"min\":176,\"max\":30028,\"num\":3,\"sum\":60004}}}"}
2025-03-17T10:18:11.045460+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206691045,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:19:07.604788+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206747604,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T10:19:07.605474+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206747605,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T10:21:11.085387+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206871085,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2882383872,\"uptime\":2026766.47,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24292860,\"nice\":16990,\"sys\":3488480,\"idle\":1991477490,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24008110,\"nice\":16540,\"sys\":3505160,\"idle\":1991703470,\"irq\":0}}],\"eventLoopStats\":{\"min\":182,\"max\":30030,\"num\":3,\"sum\":60010}}}"}
2025-03-17T10:21:11.085519+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206871085,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:21:25.204960+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206885204,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Roles for  incident: [{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}]"}
2025-03-17T10:21:25.892595+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206885892,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 8314687c-abb3-4165-bbbf-414b12112a6b-incident-info, info: {\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseId\":\"500EE00001TyZ1rYAF\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"slackChannelId\":\"C08JMPZNEGY\",\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"severity\":\"Sev2\",\"status\":\"InProgress\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T07:53:00.000Z\",\"impactEnd\":\"2025-03-17T09:08:00.000Z\",\"servicesImpacted\":\"Core Live on Falcon\",\"roles\":{\"Incident Commander\":\"Pavithra Kuppusami\"},\"category\":\"Category not found\",\"rootCause\":\"Root cause not found\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"caseNumber\":\"73104263\",\"productTag\":\"Product tag not found\",\"proximateCause\":\"Proximate cause not found\",\"customerExperience\":\"Customer Experience not found\",\"immediateResolution\":\"Immediate Resolution not found\",\"aarQuestions\":\"Aar Questions not found\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Closed By not found\"}"}
2025-03-17T10:21:25.893664+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206885893,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-incident-updated correctly routed"}
2025-03-17T10:22:35.509627+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:22:35.509737+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-to-cis-response' for consumer group='icc-reporting'"}
2025-03-17T10:22:35.509748+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:22:35.455Z\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"responses\":[{\"response\":{\"id\":\"aCqEE000000C11x0AC\",\"success\":true,\"errors\":[]},\"configurationItemId\":\"a7JEE00000J7QbT2AV\"}]}"}
2025-03-17T10:22:35.509764+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-response\"}}"}
2025-03-17T10:22:35.509825+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-response\"}}"}
2025-03-17T10:22:35.509917+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206955509,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:22:35.510149+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742206955510,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:22:35.547867+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955547,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:22:35.548061+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-incident-to-cis-export-success' for consumer group='icc-reporting'"}
2025-03-17T10:22:35.548102+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:22:35.455Z\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"responses\":[{\"response\":{\"id\":\"aCqEE000000C11x0AC\",\"success\":true,\"errors\":[]},\"configurationItemId\":\"a7JEE00000J7QbT2AV\"}],\"cis\":[{\"id\":\"d6d4dd28-3f22-4427-b9e9-134a823c8027\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"createLock\":false,\"sfid\":\"aCqEE000000C11x0AC\",\"configurationItem\":{\"id\":\"a7JEE00000J7QbT2AV\",\"name\":\"usa662s\",\"type\":\"Hyperforce Cell\",\"path\":\"Salesforce.Hyperforce.commercial1.aws-prod5-uswest2.core2.usa662s\",\"parentId\":\"a7JEE00000GHhov2AD\",\"status\":null,\"childrenCount\":-1},\"lock\":{\"createdAt\":\"2025-03-17T08:12:58.225Z\",\"modifiedAt\":\"2025-03-17T10:22:34.141Z\",\"id\":\"e6b75ce2-ca2b-46f7-8b51-6441c7b9106d\",\"status\":\"pending_response\",\"description\":\"Lock to be created for Incident: 8314687c-abb3-4165-bbbf-414b12112a6b\",\"externalId\":\"a8LEE00001Bctft2AB\"}}]}"}
2025-03-17T10:22:35.548105+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-export-success\"}}"}
2025-03-17T10:22:35.548161+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-incident-to-cis-export-success\"}}"}
2025-03-17T10:22:35.548255+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:22:35.548491+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742206955548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:22:36.006937+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206956006,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Roles for  incident: []"}
2025-03-17T10:22:40.314076+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960313,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:22:40.314204+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T10:22:40.314209+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:22:40.078Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T10:22:40.078Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}"}
2025-03-17T10:22:40.314233+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:22:40.314300+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:22:40.314439+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:22:40.314442+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response"}
2025-03-17T10:22:40.314443+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T10:22:40.314642+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T10:22:40.314993+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId ODAAAAeWWAB and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,50,50,58,52,48,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:22:40.078Z\\\",\\\"user\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_resolved\\\",\\\"timestamp\\\":\\\"2025-03-17T10:22:40.078Z\\\",\\\"recordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"incidentId\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"caseNumber\\\":\\\"73104263\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:22:40.078Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T10:22:40.078Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T10:22:40.078Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T10:22:40.078Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}}"}
2025-03-17T10:22:40.314994+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId found for [object Object]: ODAAAAeWWAB"}
2025-03-17T10:22:40.314995+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"routeEventJson for icc-gus-response and docId ODAAAAeWWAB"}
2025-03-17T10:22:40.314996+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"kafkaReportConsumer.handleGusExport for id 8314687c-abb3-4165-bbbf-414b12112a6b and docId ODAAAAeWWAB"}
2025-03-17T10:22:40.314996+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, event {\"status\":\"success\",\"event\":\"incident_resolved\",\"timestamp\":\"2025-03-17T10:22:40.078Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"}"}
2025-03-17T10:22:40.315071+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960315,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b-gus-casenumber - value: [object Promise]"}
2025-03-17T10:22:40.315216+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206960315,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"addGusExportUpdate for incidentId 8314687c-abb3-4165-bbbf-414b12112a6b, docId ODAAAAeWWAB, caseNumber 73104263"}
2025-03-17T10:22:40.315218+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206960315,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response correctly routed"}
2025-03-17T10:23:11.117421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742206991117,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2880757760,\"uptime\":2026886.5,\"totalMem\":4018151424,\"loadavg\":[0.02,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24294310,\"nice\":16990,\"sys\":3488620,\"idle\":1991595830,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24008940,\"nice\":16540,\"sys\":3505300,\"idle\":1991821960,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29344,\"num\":36,\"sum\":60014}}}"}
2025-03-17T10:23:11.117514+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742206991117,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:27:35.870984+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:27:35.870992+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='bullmq-job-completed' for consumer group='icc-reporting'"}
2025-03-17T10:27:35.870994+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:27:35.817Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\"},\"eventObj\":{\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"reminderId\":\"f4858a94-46d2-4ec6-8f49-78b8b7961d93\",\"message\":\"Send Resolved TCE\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"}},\"client\":\"icc-rest\"}"}
2025-03-17T10:27:35.870996+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T10:27:35.870997+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"bullmq-job-completed\"}}"}
2025-03-17T10:27:35.870998+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:27:35.870999+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - bullmq-job-completed"}
2025-03-17T10:27:35.870999+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type bullmq-job-completed is not consumed by icc-reporting."}
2025-03-17T10:27:35.871000+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255870,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - bullmq-job-completed correctly routed"}
2025-03-17T10:27:35.958367+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:27:35.958572+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-reminder-triggered' for consumer group='icc-reporting'"}
2025-03-17T10:27:35.958608+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:27:35.908Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\"},\"eventObj\":{\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"reminderId\":\"f4858a94-46d2-4ec6-8f49-78b8b7961d93\",\"message\":\"Send Resolved TCE\",\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"reminderData\":{\"id\":\"f4858a94-46d2-4ec6-8f49-78b8b7961d93\",\"type\":\"REMINDER\",\"active\":true,\"message\":\"Send Resolved TCE\",\"audience\":[],\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"offsetUnit\":\"m\",\"triggersAt\":\"2025-03-17T10:27:35.771Z\",\"isRecurring\":false,\"occurrences\":[],\"offsetValue\":5,\"additionalInfo\":{\"rule\":{\"name\":\"Resolved Incident Reminders\",\"type\":null},\"status\":\"Resolved\"},\"currentOccurrence\":0,\"inheritedFromTeamConfig\":true},\"slackChannelId\":\"C08JMPZNEGY\"}}"}
2025-03-17T10:27:35.958643+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T10:27:35.958684+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-reminder-triggered\"}}"}
2025-03-17T10:27:35.958792+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:27:35.958809+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-reminder-triggered"}
2025-03-17T10:27:35.958838+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-reminder-triggered is not consumed by icc-reporting."}
2025-03-17T10:27:35.958868+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207255958,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-reminder-triggered correctly routed"}
2025-03-17T10:29:11.245421+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207351245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2879315968,\"uptime\":2027246.63,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24296930,\"nice\":16990,\"sys\":3489030,\"idle\":1991951360,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24011340,\"nice\":16540,\"sys\":3505780,\"idle\":1992177430,\"irq\":0}}],\"eventLoopStats\":{\"min\":144,\"max\":30010,\"num\":3,\"sum\":60001}}}"}
2025-03-17T10:29:11.245481+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207351245,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:30:11.265415+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207411265,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2879094784,\"uptime\":2027306.65,\"totalMem\":4018151424,\"loadavg\":[0.18,0.06,0.02],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24297320,\"nice\":16990,\"sys\":3489110,\"idle\":1992010520,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24011650,\"nice\":16540,\"sys\":3505830,\"idle\":1992236730,\"irq\":0}}],\"eventLoopStats\":{\"min\":152,\"max\":30013,\"num\":3,\"sum\":60017}}}"}
2025-03-17T10:30:11.265474+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207411265,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:36:11.393415+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742207771393,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2875727872,\"uptime\":2027666.78,\"totalMem\":4018151424,\"loadavg\":[0,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24300830,\"nice\":16990,\"sys\":3489560,\"idle\":1992366030,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24014210,\"nice\":16540,\"sys\":3506250,\"idle\":1992592210,\"irq\":0}}],\"eventLoopStats\":{\"min\":95,\"max\":30029,\"num\":3,\"sum\":60027}}}"}
2025-03-17T10:36:11.393466+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742207771393,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:41:31.314150+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208091313,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T10:41:31.314972+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208091314,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T10:44:11.525401+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208251525,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2873462784,\"uptime\":2028146.91,\"totalMem\":4018151424,\"loadavg\":[0.07,0.02,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24303770,\"nice\":16990,\"sys\":3490250,\"idle\":1992840280,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24017900,\"nice\":16540,\"sys\":3506920,\"idle\":1993065920,\"irq\":0}}],\"eventLoopStats\":{\"min\":75,\"max\":30006,\"num\":3,\"sum\":60010}}}"}
2025-03-17T10:44:11.525462+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208251525,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:46:11.553405+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208371553,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2873430016,\"uptime\":2028266.94,\"totalMem\":4018151424,\"loadavg\":[0.01,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24304710,\"nice\":16990,\"sys\":3490390,\"idle\":1992958660,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24018840,\"nice\":16540,\"sys\":3507090,\"idle\":1993184500,\"irq\":0}}],\"eventLoopStats\":{\"min\":86,\"max\":30030,\"num\":3,\"sum\":60017}}}"}
2025-03-17T10:46:11.553627+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208371553,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:46:32.275825+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208392275,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Connected to Redis"}
2025-03-17T10:46:32.276580+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208392276,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Redis client ready"}
2025-03-17T10:47:06.310157+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:06.310428+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T10:47:06.310517+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:06.294Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"x5l4oi-xi20f-7avph1-fqbbyd-1742208426097\",\"client\":\"icc-ui\"}"}
2025-03-17T10:47:06.310522+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:47:06.310609+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:47:06.310685+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:06.310698+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted"}
2025-03-17T10:47:06.310725+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T10:47:06.310768+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426310,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted correctly routed"}
2025-03-17T10:47:06.327896+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426327,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:06.328145+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-created' for consumer group='icc-reporting'"}
2025-03-17T10:47:06.328245+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:06.318Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Resolved\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":[],\"requestId\":\"x5l4oi-xi20f-7avph1-fqbbyd-1742208426097\",\"client\":\"icc-ui\"}"}
2025-03-17T10:47:06.328271+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:47:06.328296+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-created\"}}"}
2025-03-17T10:47:06.328391+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:06.328391+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-created"}
2025-03-17T10:47:06.328434+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-created is not consumed by icc-reporting."}
2025-03-17T10:47:06.328435+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426328,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-created correctly routed"}
2025-03-17T10:47:06.377800+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426377,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:06.377938+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426377,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-response' for consumer group='icc-reporting'"}
2025-03-17T10:47:06.377946+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426377,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:06.328Z\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"responses\":[]}"}
2025-03-17T10:47:06.377950+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426377,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:47:06.378018+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426377,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-response\"}}"}
2025-03-17T10:47:06.378088+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426378,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:06.378370+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742208426378,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:47:06.548538+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:06.548796+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-usd-resource-deleted' for consumer group='icc-reporting'"}
2025-03-17T10:47:06.548867+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:06.538Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[{\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"modifiedAt\":\"2025-03-17T08:12:44.236Z\",\"createdBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"modifiedBy\":{\"id\":\"02badda0-dae7-4498-93ec-ab6193b2d8a1\",\"email\":\"sitereliability@salesforce.com\",\"active\":true,\"name\":\"SiteReliability\",\"firstName\":null,\"lastName\":null,\"profile\":\"icc standard user\",\"gusUserId\":\"00590000000DrNxAAK\",\"slackUserId\":null,\"createdBy\":\"jgoble@salesforce.com\",\"modifiedBy\":\"jgoble@salesforce.com\",\"createdAt\":\"2022-11-14T22:00:55.527Z\",\"modifiedAt\":\"2022-11-14T22:00:55.527Z\"},\"id\":\"9612e4ea-7f9f-4836-957a-9d07056c2237\",\"name\":\"IncidentCommander\",\"displayName\":\"Incident Commander\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"externalGusId\":null}],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"ids\":[]},\"requestId\":\"x5l4oi-xi20f-7avph1-fqbbyd-1742208426097\",\"client\":\"icc-ui\"}"}
2025-03-17T10:47:06.548873+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:47:06.548910+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-usd-resource-deleted\"}}"}
2025-03-17T10:47:06.549003+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426548,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:06.549027+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208426549,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted"}
2025-03-17T10:47:06.549051+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426549,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event type icc-usd-resource-deleted is not consumed by icc-reporting."}
2025-03-17T10:47:06.549072+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208426549,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-usd-resource-deleted correctly routed"}
2025-03-17T10:47:07.576694+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208427576,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Roles for  incident: []"}
2025-03-17T10:47:08.420449+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208428420,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"RedisService: saveIncidentInfo, key: 8314687c-abb3-4165-bbbf-414b12112a6b-incident-info, info: {\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseId\":\"500EE00001TyZ1rYAF\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"slackChannelId\":\"C08JMPZNEGY\",\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"impact\":\"PerformanceDegradation\",\"impactStart\":\"2025-03-17T07:53:00.000Z\",\"impactEnd\":\"2025-03-17T09:08:00.000Z\",\"servicesImpacted\":\"Core Live on Falcon\",\"roles\":{},\"category\":\"Application\",\"rootCause\":\"Change\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"caseNumber\":\"73104263\",\"productTag\":\"a1aEE00000053lFYAQ\",\"proximateCause\":\"The issue with row lock waits\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\nFOR NO KEY UPDATE WAIT 10\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\",\"customerExperience\":\"slow app loads, performance degradations\",\"immediateResolution\":\"Issue self-resolved\",\"aarQuestions\":\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\",\"closingComments\":\"Closing comments not found\",\"closedBy\":\"Pavithra Kuppusami\"}"}
2025-03-17T10:47:08.421670+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208428421,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages Adding Slack messages to docId ODAAAAeWWAB..."}
2025-03-17T10:47:08.991681+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208428991,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02DWLVTKB9"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.002574+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429002,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02GXBX8PUZ"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.023645+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429023,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U02Q288NF7W"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.027251+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429026,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U032QNNKMT7"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.030022+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429029,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U01UPBJAV5F"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.034719+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429034,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U06Q54CS88M"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:09.037243+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208429037,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"message":"Request failed with status code 404","name":"AxiosError","config":{"transitional":{"silentJSONParsing":true,"forcedJSONParsing":true,"clarifyTimeoutError":false},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1,"maxBodyLength":-1,"env":{},"headers":{"Accept":"application/json, text/plain, */*","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiaWNjLXJlcG9ydGluZy11c2VyIiwiZW1haWwiOiJpY2NAcmVwb3J0aW5nLmFwcCIsImlzU3lzdGVtVXNlciI6dHJ1ZSwiaWF0IjoxNzQyMTY5MTc4LCJhdWQiOiJpY2MtcmVzdC1hcGkiLCJpc3MiOiJpY2MtcmVwb3J0aW5nIn0.ZjOLkNz068U_szSmLqVDE7gWNvZmWLVWJ3FQTvw5Sos","User-Agent":"axios/0.27.2"},"method":"get","url":"https://icc-rest-production.herokuapp.com/api/v1/users/slack/U03LNRS637U"},"code":"ERR_BAD_REQUEST","status":404},"msg":"ICC-Rest has not returned user from Slack Id"}
2025-03-17T10:47:10.914770+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430914,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages remove redis keys for docId ODAAAAeWWAB"}
2025-03-17T10:47:10.916690+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"exportSlackMessages produceEvent for docId ODAAAAeWWAB"}
2025-03-17T10:47:10.916691+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"produceEvent with link https://salesforce.quip.com/m6mBA2aDHeqm for event 8314687c-abb3-4165-bbbf-414b12112a6b and case 73104263"}
2025-03-17T10:47:10.916756+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricRequestExecute] - icc-report-finished - undefined"}
2025-03-17T10:47:10.916760+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:10.916828+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:10.916919+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:10.917075+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430916,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key '8314687c-abb3-4165-bbbf-414b12112a6b' to topic 'incident': {\"createdAt\":\"2025-03-17T10:47:10.916Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"closingDetails\":{\"productTag\":\"a1aEE00000053lFYAQ\",\"detectedByServiceOwner\":false,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The issue with row lock waits\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\nFOR NO KEY UPDATE WAIT 10\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\",\"customerExperience\":\"slow app loads, performance degradations\",\"immediateResolution\":\"Issue self-resolved\",\"aarQuestions\":\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"LWPCS - Lightning Web Platform Customer Success\",\"closedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"createdBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"modifiedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:47:06.246Z\",\"modifiedAt\":\"2025-03-17T10:47:06.246Z\",\"id\":\"0525a46c-bfe8-4b27-82ac-ac8cde0598aa\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/m6mBA2aDHeqm\"}}"}
2025-03-17T10:47:10.917148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key '8314687c-abb3-4165-bbbf-414b12112a6b' to Kafka topic 'incident'"}
2025-03-17T10:47:10.917148+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.iccMetricDurationTime] - icc-report-finished - undefined"}
2025-03-17T10:47:10.917181+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.2165369987487793\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:10.917234+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"duration\",\"value\":\"0.2165369987487793\",\"topicName\":\"metrics\",\"producerConsumer\":\"producer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:10.917249+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:10.917302+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208430917,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-incident-closed correctly routed"}
2025-03-17T10:47:11.118314+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431117,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:11.118317+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-report-finished' for consumer group='icc-reporting'"}
2025-03-17T10:47:11.118427+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:10.916Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"closingDetails\":{\"productTag\":\"a1aEE00000053lFYAQ\",\"detectedByServiceOwner\":false,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The issue with row lock waits\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\nFOR NO KEY UPDATE WAIT 10\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\",\"customerExperience\":\"slow app loads, performance degradations\",\"immediateResolution\":\"Issue self-resolved\",\"aarQuestions\":\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"LWPCS - Lightning Web Platform Customer Success\",\"closedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"createdBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"modifiedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:47:06.246Z\",\"modifiedAt\":\"2025-03-17T10:47:06.246Z\",\"id\":\"0525a46c-bfe8-4b27-82ac-ac8cde0598aa\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/m6mBA2aDHeqm\"}}"}
2025-03-17T10:47:11.118431+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:11.118547+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-report-finished\"}}"}
2025-03-17T10:47:11.118669+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:11.118670+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-report-finished"}
2025-03-17T10:47:11.118716+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-report-finished is now being processed."}
2025-03-17T10:47:11.118852+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431118,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T10:47:11.119290+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431119,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,45,102,105,110,105,115,104,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,52,55,58,49,48,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,114,101,112,111,114,116,105,110,103]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:47:10.916Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"icc-73104263\\\"},\\\"subject\\\":\\\"APT Degradation on USA662s, core2, aws-prod5-uswest2\\\",\\\"description\\\":\\\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\\\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\\\n\\\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\\\",\\\"detectedTime\\\":\\\"2025-03-17T08:06:00.000Z\\\",\\\"severity\\\":\\\"Sev2\\\",\\\"status\\\":\\\"Closed\\\",\\\"origin\\\":\\\"Alert\\\",\\\"originReference\\\":\\\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\\\",\\\"impact\\\":[\\\"PerformanceDegradation\\\"],\\\"impactStartTime\\\":\\\"2025-03-17T07:53:00.000Z\\\",\\\"impactEndTime\\\":\\\"2025-03-17T09:08:00.000Z\\\",\\\"bridge\\\":{\\\"url\\\":\\\"https://meet.google.com/hfi-qyjv-nzz\\\",\\\"additionalJoiningInfo\\\":\\\"Join By Phone:\\\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\\\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\\\n\\\"},\\\"roles\\\":[],\\\"category\\\":\\\"Application\\\",\\\"rootCause\\\":\\\"Change\\\",\\\"createdAt\\\":\\\"2025-03-17T08:12:44.236Z\\\",\\\"closingDetails\\\":{\\\"productTag\\\":\\\"a1aEE00000053lFYAQ\\\",\\\"detectedByServiceOwner\\\":false,\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"proximateCause\\\":\\\"The issue with row lock waits\\\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\\\nFOR NO KEY UPDATE WAIT 10\\\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\\\",\\\"customerExperience\\\":\\\"slow app loads, performance degradations\\\",\\\"immediateResolution\\\":\\\"Issue self-resolved\\\",\\\"aarQuestions\\\":\\\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\\\",\\\"changes\\\":\\\"\\\",\\\"breakFix\\\":\\\"\\\",\\\"productTagName\\\":\\\"LWPCS - Lightning Web Platform Customer Success\\\",\\\"closedBy\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"incidentId\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"createdBy\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"modifiedBy\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"active\\\":true,\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"firstName\\\":\\\"Pavithra\\\",\\\"lastName\\\":\\\"Kuppusami\\\",\\\"profile\\\":\\\"icc standard user\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"createdBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"modifiedBy\\\":\\\"pavithra.kuppusami@salesforce.com\\\",\\\"createdAt\\\":\\\"2022-04-28T15:47:19.245Z\\\",\\\"modifiedAt\\\":\\\"2022-04-28T15:47:19.245Z\\\"},\\\"closingComments\\\":null,\\\"createdAt\\\":\\\"2025-03-17T10:47:06.246Z\\\",\\\"modifiedAt\\\":\\\"2025-03-17T10:47:06.246Z\\\",\\\"id\\\":\\\"0525a46c-bfe8-4b27-82ac-ac8cde0598aa\\\"},\\\"scrumTeamId\\\":\\\"a0090000000vv4hAAA\\\",\\\"team\\\":\\\"Site Reliability\\\",\\\"caseNumber\\\":\\\"73104263\\\",\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"servicesImpacted\\\":[\\\"Core Live on Falcon\\\"],\\\"prbNumber\\\":{\\\"createdByIcc\\\":true},\\\"quipDocLink\\\":\\\"https://salesforce.quip.com/m6mBA2aDHeqm\\\",\\\"engagedAt\\\":\\\"2025-03-17T08:12:58.310Z\\\",\\\"lastAttemptedForceExport\\\":\\\"\\\",\\\"lastExportStatus\\\":\\\"success\\\",\\\"gusExportTimestamp\\\":\\\"2025-03-17T10:22:40.078Z\\\"},\\\"eventObj\\\":{\\\"link\\\":\\\"https://salesforce.quip.com/m6mBA2aDHeqm\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:47:10.916Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"closingDetails\":{\"productTag\":\"a1aEE00000053lFYAQ\",\"detectedByServiceOwner\":false,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The issue with row lock waits\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\nFOR NO KEY UPDATE WAIT 10\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\",\"customerExperience\":\"slow app loads, performance degradations\",\"immediateResolution\":\"Issue self-resolved\",\"aarQuestions\":\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"LWPCS - Lightning Web Platform Customer Success\",\"closedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"createdBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"modifiedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.c
2025-03-17T10:47:11.119290+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] om\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:47:06.246Z\",\"modifiedAt\":\"2025-03-17T10:47:06.246Z\",\"id\":\"0525a46c-bfe8-4b27-82ac-ac8cde0598aa\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/m6mBA2aDHeqm\"}},\"encoder\":\"json\",\"eventType\":\"icc-report-finished\",\"payload\":{\"createdAt\":\"2025-03-17T10:47:10.916Z\",\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"icc-73104263\"},\"subject\":\"APT Degradation on USA662s, core2, aws-prod5-uswest2\",\"description\":\"Sherlock detected an APT Incident for Cell: usa662s, FI: AWS-PROD5-USWEST2. \\nAnomalies detected in the timeframe 2025-03-17 08:01:00.000 - 2025-03-17 08:03:00.000.\\n\\nCase created using .case by @Pavithra Kuppusami in #cp-sre-alerts for usa662s\",\"detectedTime\":\"2025-03-17T08:06:00.000Z\",\"severity\":\"Sev2\",\"status\":\"Closed\",\"origin\":\"Alert\",\"originReference\":\"https://salesforce.pagerduty.com/incidents/Q15U2Z6PMK3QFM\",\"impact\":[\"PerformanceDegradation\"],\"impactStartTime\":\"2025-03-17T07:53:00.000Z\",\"impactEndTime\":\"2025-03-17T09:08:00.000Z\",\"bridge\":{\"url\":\"https://meet.google.com/hfi-qyjv-nzz\",\"additionalJoiningInfo\":\"Join By Phone:\\nUS number: tel:+1-224-324-4544, pin: 110097939; \\nFind numbers in more regions on https://tel.meet/hfi-qyjv-nzz?pin=1325516168543, pin: 1325516168543\\n\"},\"roles\":[],\"category\":\"Application\",\"rootCause\":\"Change\",\"createdAt\":\"2025-03-17T08:12:44.236Z\",\"closingDetails\":{\"productTag\":\"a1aEE00000053lFYAQ\",\"detectedByServiceOwner\":false,\"prbNumber\":{\"createdByIcc\":true},\"proximateCause\":\"The issue with row lock waits\\nSELECT /*SFSQL at cMetaAccess.sql(3c6e079):check_access_nc:714*/\\nFOR NO KEY UPDATE WAIT 10\\nwhich were caused due to timing of access as summarized in this Quip Doc first showed impact on Product Versions > 254.10.3 as noted in PRB-0027052's summary here. We do not have an exact PR which was rolled out in that version yet but we will consult with SLDS Surge Team (represented by @Neeraj Agicha) to get more information as the specific piece of code that has been identified as being the cause of this regression was updated by their team.\",\"customerExperience\":\"slow app loads, performance degradations\",\"immediateResolution\":\"Issue self-resolved\",\"aarQuestions\":\"1. Why is the fix-forward/ remediation plan not yet ready? as this issue is seen on multiple instances.\\n2. RCA on the impact, as there were multiple Jobs that contributed to the sql_wait_lock event during the impact.\",\"changes\":\"\",\"breakFix\":\"\",\"productTagName\":\"LWPCS - Lightning Web Platform Customer Success\",\"closedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"createdBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"modifiedBy\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"email\":\"pavithra.kuppusami@salesforce.com\",\"active\":true,\"name\":\"Pavithra Kuppusami\",\"firstName\":\"Pavithra\",\"lastName\":\"Kuppusami\",\"profile\":\"icc standard user\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"createdBy\":\"pavithra.kuppusami@salesforce.com\",\"modifiedBy\":\"pavithra.kuppusami@salesforce.com\",\"createdAt\":\"2022-04-28T15:47:19.245Z\",\"modifiedAt\":\"2022-04-28T15:47:19.245Z\"},\"closingComments\":null,\"createdAt\":\"2025-03-17T10:47:06.246Z\",\"modifiedAt\":\"2025-03-17T10:47:06.246Z\",\"id\":\"0525a46c-bfe8-4b27-82ac-ac8cde0598aa\"},\"scrumTeamId\":\"a0090000000vv4hAAA\",\"team\":\"Site Reliability\",\"caseNumber\":\"73104263\",\"caseRecordId\":\"500EE00001TyZ1rYAF\",\"servicesImpacted\":[\"Core Live on Falcon\"],\"prbNumber\":{\"createdByIcc\":true},\"quipDocLink\":\"https://salesforce.quip.com/m6mBA2aDHeqm\",\"engagedAt\":\"2025-03-17T08:12:58.310Z\",\"lastAttemptedForceExport\":\"\",\"lastExportStatus\":\"success\",\"gusExportTimestamp\":\"2025-03-17T10:22:40.078Z\"},\"eventObj\":{\"link\":\"https://salesforce.quip.com/m6mBA2aDHeqm\"}}}"}
2025-03-17T10:47:11.119323+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431119,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:11.119376+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208431119,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-report-finished does not have the necessary information to create a document"}
2025-03-17T10:47:11.119424+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431119,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-report-finished correctly routed"}
2025-03-17T10:47:11.553407+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431553,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2873438208,\"uptime\":2028326.94,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24305610,\"nice\":16990,\"sys\":3490450,\"idle\":1993017680,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24019330,\"nice\":16540,\"sys\":3507180,\"idle\":1993243650,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":24672,\"num\":132,\"sum\":59947}}}"}
2025-03-17T10:47:11.553551+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431553,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:11.833648+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:11.833778+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-gus-response' for consumer group='icc-reporting'"}
2025-03-17T10:47:11.833795+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:47:11.782Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:47:11.782Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}"}
2025-03-17T10:47:11.833808+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:47:11.833862+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-gus-response\"}}"}
2025-03-17T10:47:11.833958+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:11.833980+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response"}
2025-03-17T10:47:11.833982+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431833,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-gus-response is now being processed."}
2025-03-17T10:47:11.834123+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431834,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 8314687c-abb3-4165-bbbf-414b12112a6b - value: [object Promise]"}
2025-03-17T10:47:11.834417+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431834,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,114,101,115,112,111,110,115,101]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,52,55,58,49,49,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,103,117,115,45,101,120,112,111,114,116]}}],\"key\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:47:11.782Z\\\",\\\"user\\\":{\\\"id\\\":\\\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\\\",\\\"gusUserId\\\":\\\"005B0000006phyJIAQ\\\",\\\"slackUserId\\\":\\\"U01G5MUL669\\\",\\\"name\\\":\\\"Pavithra Kuppusami\\\",\\\"email\\\":\\\"pavithra.kuppusami@salesforce.com\\\"},\\\"incident\\\":{\\\"id\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"C08JMPZNEGY\\\",\\\"name\\\":\\\"\\\"},\\\"caseRecordId\\\":\\\"500EE00001TyZ1rYAF\\\"},\\\"eventObj\\\":{\\\"status\\\":\\\"success\\\",\\\"event\\\":\\\"incident_closed\\\",\\\"timestamp\\\":\\\"2025-03-17T10:47:11.782Z\\\",\\\"recordId\\\":\\\"500EE00001TyZ1rYAF\\\",\\\"incidentId\\\":\\\"8314687c-abb3-4165-bbbf-414b12112a6b\\\",\\\"caseNumber\\\":\\\"73104263\\\"},\\\"changes\\\":{}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:47:11.782Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:47:11.782Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}},\"encoder\":\"json\",\"eventType\":\"icc-gus-response\",\"payload\":{\"createdAt\":\"2025-03-17T10:47:11.782Z\",\"user\":{\"id\":\"5aed21f9-3f31-4016-a7f0-600fa6ebea8d\",\"gusUserId\":\"005B0000006phyJIAQ\",\"slackUserId\":\"U01G5MUL669\",\"name\":\"Pavithra Kuppusami\",\"email\":\"pavithra.kuppusami@salesforce.com\"},\"incident\":{\"id\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"slackChannel\":{\"id\":\"C08JMPZNEGY\",\"name\":\"\"},\"caseRecordId\":\"500EE00001TyZ1rYAF\"},\"eventObj\":{\"status\":\"success\",\"event\":\"incident_closed\",\"timestamp\":\"2025-03-17T10:47:11.782Z\",\"recordId\":\"500EE00001TyZ1rYAF\",\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\",\"caseNumber\":\"73104263\"},\"changes\":{}}}"}
2025-03-17T10:47:11.834419+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208431834,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:11.834434+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208431834,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-gus-response does not have the necessary information to create a document"}
2025-03-17T10:47:11.834435+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208431834,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 8314687c-abb3-4165-bbbf-414b12112a6b - icc-gus-response correctly routed"}
2025-03-17T10:47:21.269768+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208441269,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 8314687c-abb3-4165-bbbf-414b12112a6b"}
2025-03-17T10:47:21.269841+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208441269,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-ctc-lock-response' for consumer group='icc-reporting'"}
2025-03-17T10:47:21.269842+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208441269,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"locks\":[],\"incidentId\":\"8314687c-abb3-4165-bbbf-414b12112a6b\"}"}
2025-03-17T10:47:21.269858+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208441269,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T10:47:21.269926+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208441269,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-ctc-lock-response\"}}"}
2025-03-17T10:47:21.270071+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208441270,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:47:21.270288+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":50,"time":1742208441270,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","err":{"type":"TypeError","message":"Cannot read properties of undefined (reading 'id')","stack":"TypeError: Cannot read properties of undefined (reading 'id')\n    at KafkaConsumer.<anonymous> (/app/build/kafka/kafka.report.consumer.js:43:109)\n    at KafkaConsumer.emit (node:events:519:28)\n    at readCallback (/app/node_modules/node-rdkafka/lib/kafka-consumer.js:448:12)"},"msg":"[Unhandled Error] potentially crashes the app, please investigate and handle it accordingly"}
2025-03-17T10:48:11.581409+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208491581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2873438208,\"uptime\":2028386.96,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24306040,\"nice\":16990,\"sys\":3490560,\"idle\":1993076830,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24019720,\"nice\":16540,\"sys\":3507280,\"idle\":1993303030,\"irq\":0}}],\"eventLoopStats\":{\"min\":0,\"max\":29905,\"num\":88,\"sum\":60016}}}"}
2025-03-17T10:48:11.581472+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208491581,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:49:11.609774+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208551609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2860556288,\"uptime\":2028446.99,\"totalMem\":4018151424,\"loadavg\":[0,0,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24306590,\"nice\":16990,\"sys\":3490640,\"idle\":1993135780,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24020240,\"nice\":16540,\"sys\":3507390,\"idle\":1993362060,\"irq\":0}}],\"eventLoopStats\":{\"min\":94,\"max\":30029,\"num\":3,\"sum\":60027}}}"}
2025-03-17T10:49:11.609780+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208551609,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:50:03.285706+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received kafka message with key: 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:50:03.285855+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer consumed event with eventType='icc-bridge-folder-created' for consumer group='icc-reporting'"}
2025-03-17T10:50:03.285856+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Consumer is returning ProcessedKafkaEvent {\"createdAt\":\"2025-03-17T10:50:02.247Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"12PlRE3l83wLAH8zaccrly3sLocY0RSTx\"}}"}
2025-03-17T10:50:03.285859+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"[Metrics.publishMetric] - Publishing metric {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-folder-created\"}}"}
2025-03-17T10:50:03.285936+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"kafka_event\",\"data\":{\"type\":\"request\",\"value\":1,\"topicName\":\"incident\",\"producerConsumer\":\"consumer\",\"messageType\":\"icc-bridge-folder-created\"}}"}
2025-03-17T10:50:03.286026+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}
2025-03-17T10:50:03.286027+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603285,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Received JSON event: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-bridge-folder-created"}
2025-03-17T10:50:03.286030+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event with type icc-bridge-folder-created is now being processed."}
2025-03-17T10:50:03.286230+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"CacheServiceFacade.getValue key: 13054fa1-b6e2-45d6-8afe-0c7523384b29 - value: [object Promise]"}
2025-03-17T10:50:03.286611+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Start routing Incident to CI for docId null and event {\"headers\":[{\"icc-kafka-encoder\":{\"type\":\"Buffer\",\"data\":[106,115,111,110]}},{\"icc-event-type\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,98,114,105,100,103,101,45,102,111,108,100,101,114,45,99,114,101,97,116,101,100]}},{\"timestamp\":{\"type\":\"Buffer\",\"data\":[77,111,110,44,32,49,55,32,77,97,114,32,50,48,50,53,32,49,48,58,53,48,58,48,50,32,71,77,84]}},{\"event-source\":{\"type\":\"Buffer\",\"data\":[105,99,99,45,98,114,105,100,103,101]}}],\"key\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"messageAsString\":\"{\\\"createdAt\\\":\\\"2025-03-17T10:50:02.247Z\\\",\\\"incident\\\":{\\\"id\\\":\\\"13054fa1-b6e2-45d6-8afe-0c7523384b29\\\",\\\"slackChannel\\\":{\\\"id\\\":\\\"NA\\\",\\\"name\\\":\\\"NA\\\"}},\\\"eventObj\\\":{\\\"folderId\\\":\\\"12PlRE3l83wLAH8zaccrly3sLocY0RSTx\\\"}}\",\"messageAsJson\":{\"createdAt\":\"2025-03-17T10:50:02.247Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"12PlRE3l83wLAH8zaccrly3sLocY0RSTx\"}},\"encoder\":\"json\",\"eventType\":\"icc-bridge-folder-created\",\"payload\":{\"createdAt\":\"2025-03-17T10:50:02.247Z\",\"incident\":{\"id\":\"13054fa1-b6e2-45d6-8afe-0c7523384b29\",\"slackChannel\":{\"id\":\"NA\",\"name\":\"NA\"}},\"eventObj\":{\"folderId\":\"12PlRE3l83wLAH8zaccrly3sLocY0RSTx\"}}}"}
2025-03-17T10:50:03.286612+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"handleQueueEditEvents docId not found for 13054fa1-b6e2-45d6-8afe-0c7523384b29"}
2025-03-17T10:50:03.286616+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":40,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event icc-bridge-folder-created does not have the necessary information to create a document"}
2025-03-17T10:50:03.286616+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208603286,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-reporting","msg":"Event 13054fa1-b6e2-45d6-8afe-0c7523384b29 - icc-bridge-folder-created correctly routed"}
2025-03-17T10:51:11.637496+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":20,"time":1742208671637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer is producing message with key 'icc-reporting' to topic 'metrics': {\"service\":\"icc-reporting\",\"version\":\"1.23.2\",\"metric\":\"availability\",\"data\":{\"freeMem\":2865098752,\"uptime\":2028567.02,\"totalMem\":4018151424,\"loadavg\":[0.03,0.01,0],\"cpus\":[{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24307310,\"nice\":16990,\"sys\":3490870,\"idle\":1993254100,\"irq\":0}},{\"model\":\"Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz\",\"speed\":0,\"times\":{\"user\":24021230,\"nice\":16540,\"sys\":3507610,\"idle\":1993480430,\"irq\":0}}],\"eventLoopStats\":{\"min\":123,\"max\":30010,\"num\":3,\"sum\":60025}}}"}
2025-03-17T10:51:11.637574+00:00 f995e76c-45ce-47ba-9e5e-2aa8f396323b app[worker.1] {"level":30,"time":1742208671637,"pid":30,"hostname":"dyno-247e3aab-0e12-45b9-a84f-c91408696d40","name":"icc-common","msg":"Kafka Producer successfully sent event with key 'icc-reporting' to Kafka topic 'metrics'"}