From 41fa1859704b393499d400cf987f16b609f90794 Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Mon, 4 Sep 2023 16:22:37 -0300 Subject: [PATCH 1/7] Nest TopicExtractor to SimpleWriter This class was design only for working with SimpleWriter, so nesting it makes this clear. --- .../simple_writer/topic_extractor.rb | 55 +++++++++++++++++++ lib/gold_miner/topic_extractor.rb | 51 ----------------- .../simple_writer}/topic_extractor_spec.rb | 2 +- 3 files changed, 56 insertions(+), 52 deletions(-) create mode 100644 lib/gold_miner/blog_post/simple_writer/topic_extractor.rb delete mode 100644 lib/gold_miner/topic_extractor.rb rename spec/gold_miner/{ => blog_post/simple_writer}/topic_extractor_spec.rb (96%) diff --git a/lib/gold_miner/blog_post/simple_writer/topic_extractor.rb b/lib/gold_miner/blog_post/simple_writer/topic_extractor.rb new file mode 100644 index 0000000..b576a92 --- /dev/null +++ b/lib/gold_miner/blog_post/simple_writer/topic_extractor.rb @@ -0,0 +1,55 @@ +module GoldMiner + class BlogPost + class SimpleWriter + module TopicExtractor + LANGUAGE_MATCHERS = { + "Ruby" => ["ruby", "ruby on rails"], + "Elixir" => ["elixir"], + "JavaScript" => ["javascript", "js", "node", "nodejs", "yarn", "npm"], + "TypeScript" => ["typescript", "ts"], + "SQL" => ["sql"], + "CSS" => ["css"] + }.freeze + TOOL_MATCHERS = { + "Ruby on Rails" => ["ruby on rails"], + "React" => ["react", "reactjs"], + "React Native" => ["react native"], + "Tailwind" => ["tailwind css", "tailwindcss", "tailwind"] + } + TECHNIQUE_MATCHERS = { + "Refactoring" => ["refactor", "refactoring"], + "Testing" => ["test", "tests", "testing"] + }.freeze + PARDIGM_MATCHERS = { + "Functional Programming" => ["functional programming"], + "OOP" => ["object oriented programming", "oop"] + }.freeze + OTHER_MATCHERS = { + "TIL" => ["til", "today i learned", "today i learnt"], + "Tip" => ["tip", "tips"] + } + TOPIC_MATCHERS = { + **LANGUAGE_MATCHERS, + **TECHNIQUE_MATCHERS, + **TOOL_MATCHERS, + **PARDIGM_MATCHERS, + **OTHER_MATCHERS + }.freeze + + def self.call(message_text) + words = message_text.downcase.split(/\W/) + sanitized_text = words.join(" ") + topics = Set[] + + TOPIC_MATCHERS.each do |topic_label, topic_matchers| + topic_matchers.each do |topic_matcher| + topics << topic_label if sanitized_text.match?(/\b#{topic_matcher}\b/) + end + end + + topics.to_a + end + end + end + end +end diff --git a/lib/gold_miner/topic_extractor.rb b/lib/gold_miner/topic_extractor.rb deleted file mode 100644 index 1a7591f..0000000 --- a/lib/gold_miner/topic_extractor.rb +++ /dev/null @@ -1,51 +0,0 @@ -module GoldMiner - module TopicExtractor - LANGUAGE_MATCHERS = { - "Ruby" => ["ruby", "ruby on rails"], - "Elixir" => ["elixir"], - "JavaScript" => ["javascript", "js", "node", "nodejs", "yarn", "npm"], - "TypeScript" => ["typescript", "ts"], - "SQL" => ["sql"], - "CSS" => ["css"] - }.freeze - TOOL_MATCHERS = { - "Ruby on Rails" => ["ruby on rails"], - "React" => ["react", "reactjs"], - "React Native" => ["react native"], - "Tailwind" => ["tailwind css", "tailwindcss", "tailwind"] - } - TECHNIQUE_MATCHERS = { - "Refactoring" => ["refactor", "refactoring"], - "Testing" => ["test", "tests", "testing"] - }.freeze - PARDIGM_MATCHERS = { - "Functional Programming" => ["functional programming"], - "OOP" => ["object oriented programming", "oop"] - }.freeze - OTHER_MATCHERS = { - "TIL" => ["til", "today i learned", "today i learnt"], - "Tip" => ["tip", "tips"] - } - TOPIC_MATCHERS = { - **LANGUAGE_MATCHERS, - **TECHNIQUE_MATCHERS, - **TOOL_MATCHERS, - **PARDIGM_MATCHERS, - **OTHER_MATCHERS - }.freeze - - def self.call(message_text) - words = message_text.downcase.split(/\W/) - sanitized_text = words.join(" ") - topics = Set[] - - TOPIC_MATCHERS.each do |topic_label, topic_matchers| - topic_matchers.each do |topic_matcher| - topics << topic_label if sanitized_text.match?(/\b#{topic_matcher}\b/) - end - end - - topics.to_a - end - end -end diff --git a/spec/gold_miner/topic_extractor_spec.rb b/spec/gold_miner/blog_post/simple_writer/topic_extractor_spec.rb similarity index 96% rename from spec/gold_miner/topic_extractor_spec.rb rename to spec/gold_miner/blog_post/simple_writer/topic_extractor_spec.rb index 99ce006..438ac57 100644 --- a/spec/gold_miner/topic_extractor_spec.rb +++ b/spec/gold_miner/blog_post/simple_writer/topic_extractor_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" -RSpec.describe GoldMiner::TopicExtractor do +RSpec.describe GoldMiner::BlogPost::SimpleWriter::TopicExtractor do describe "#call" do it "extracts topics from a message text" do message = <<~MARKDOWN From de667a2f2f25f527415f5cad642093836192d156 Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Tue, 5 Sep 2023 11:38:05 -0300 Subject: [PATCH 2/7] Rewrite Slack::Client This class used to know too much about the app-specific needs. It's now a thin wrapper around the Slack API client gem. A major change though, is that it automatically fetches the message author real name, so it issues an additional API call for each message. To remedy the performance hits, all the author names are requested in parallel, with a cache to avoid unnecessary requests. --- lib/gold_miner/slack/client.rb | 93 ++++------ spec/gold_miner/slack/client_spec.rb | 250 +++++++++++++-------------- spec/spec_helper.rb | 6 + 3 files changed, 158 insertions(+), 191 deletions(-) diff --git a/lib/gold_miner/slack/client.rb b/lib/gold_miner/slack/client.rb index 4f1ddc5..787d2a9 100644 --- a/lib/gold_miner/slack/client.rb +++ b/lib/gold_miner/slack/client.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "dry/monads" -require "async" require "slack-ruby-client" module GoldMiner @@ -10,8 +9,8 @@ class Slack::Client extend Dry::Monads[:result] - def self.build(api_token:, slack_client: ::Slack::Web::Client, author_config: AuthorConfig.default) - client = new(api_token, slack_client, author_config) + def self.build(api_token:, slack_client: ::Slack::Web::Client) + client = new(api_token, slack_client) begin client.auth_test @@ -24,81 +23,61 @@ def self.build(api_token:, slack_client: ::Slack::Web::Client, author_config: Au end end - def initialize(api_token, slack_client, author_config) + def initialize(api_token, slack_client) @slack = slack_client.new(token: api_token) - @author_config = author_config + @user_name_cache = {} end def auth_test @slack.auth_test end - def search_interesting_messages_in(channel) - interesting_messages = interesting_messages_query(channel) - Sync do - til_messages_task = Async do - extract_messages_from_result( - @slack.search_messages(query: interesting_messages.with_topic("TIL")) - ) - end - tip_messages_task = Async do - extract_messages_from_result( - @slack.search_messages(query: interesting_messages.with_topic("tip")) - ) - end - golden_messages_task = Async do - extract_messages_from_result( - @slack.search_messages(query: interesting_messages.with_reaction(GOLD_EMOJI)) - ) - end - - [til_messages_task, tip_messages_task, golden_messages_task] - .flat_map(&:wait) - .uniq { |message| message[:permalink] } - end + def search_messages(query) + @slack + .search_messages(query:) + .tap { |response| + warn_on_multiple_pages(response) + fetch_author_names(response) + } end private_class_method :new private - def interesting_messages_query(channel) - MessagesQuery - .new - .on_channel(channel) - .sent_after_last_friday + # For simplicity, I'm not handling API pagination yet + def warn_on_multiple_pages(result) + if result.messages.paging.pages > 1 + warn "[WARNING] Found more than one page of results, only the first page will be processed" + end end - def extract_messages_from_result(result) - warn_on_multiple_pages(result) + # Unfortunately, the Slack API doesn't return the real name of the + # author of a message, so we need to make an additional API call for + # each message. + def fetch_author_names(response) + Sync do + author_names = response.messages.matches.map { |message| + Async { [message.id, real_name_for(message.user)] } + }.map(&:wait).to_h - result.messages.matches.map do |match| - Slack::Message.new( - text: match.text, - author: build_author(match), - permalink: match.permalink - ) + response.messages.matches.each { |message| + message.author_real_name = author_names[message.id] + } end end - def build_author(message) - Slack::User.new( - id: message.user, - name: fetch_author(message.user), - username: message.username, - link: @author_config.link_for(message.username) - ) + def real_name_for(user_id) + @user_name_cache[user_id] ||= + user_info(user_id) + .user + .profile + .real_name + .tap { |name| @user_name_cache[user_id] = name } end - def fetch_author(user_id) - @slack.users_info(user: user_id).user.profile.real_name - end - - # For simplicity, I'm not handling API pagination yet - def warn_on_multiple_pages(result) - if result.messages.paging.pages > 1 - warn "[WARNING] Found more than one page of results, only the first page will be processed" - end + def user_info(user_id) + @slack.users_info(user: user_id) end end end diff --git a/spec/gold_miner/slack/client_spec.rb b/spec/gold_miner/slack/client_spec.rb index 07516f8..d01b7ec 100644 --- a/spec/gold_miner/slack/client_spec.rb +++ b/spec/gold_miner/slack/client_spec.rb @@ -45,131 +45,138 @@ end end - describe "#search_interesting_messages_in" do - it "returns uniq interesting messages sent on dev channel since last friday" do - travel_to "2022-10-07" do - token = "valid-token" - user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") - user2 = TestFactories.create_slack_user(id: "user-id-2", name: "User 2", username: "username-2") - user3 = TestFactories.create_slack_user(id: "user-id-3", name: "User 3", username: "username-3") - - stub_slack_auth_test_request(status: 200, token: token) - stub_slack_message_search_request( - query: "TIL in:dev after:2022-09-30", - body: { - "ok" => true, - "messages" => { - "matches" => [ - {"text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"}, - {"text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"} - ], - "paging" => {"pages" => 1} - } - } - ) - stub_slack_message_search_request( - query: "tip in:dev after:2022-09-30", - body: { - "ok" => true, - "messages" => { - "matches" => [ - {"text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"}, - {"text" => "Ruby tip: have fun!", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-3-permalink.com"} - ], - "paging" => {"pages" => 1} - } + describe "#search_messages" do + it "returns a list of messages matching the query, including their respective author names" do + token = "valid-token" + stub_slack_auth_test_request(status: 200, token: token) + search_query = "tip in:dev" + user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") + user2 = TestFactories.create_slack_user(id: "user-id-2", name: "User 2", username: "username-2") + msg1 = {"id" => "msg1", "text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"} + msg2 = {"id" => "msg2", "text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"} + stub_slack_message_search_request( + query: search_query, + body: { + "ok" => true, + "messages" => { + "matches" => [msg1, msg2], + "paging" => {"pages" => 1} } - ) - stub_slack_message_search_request( - query: "in:dev after:2022-09-30 has::rupee-gold:", - body: { - "ok" => true, - "messages" => { - "matches" => [ - {"text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"}, - {"text" => "CSS clamp() is so cool!", "user" => user3.id, "username" => user3.username, "permalink" => "https:///message-4-permalink.com"} - ], - "paging" => {"pages" => 1} - } + } + ) + stub_slack_users_info_request( + token: token, + user_id: user1.id, + body: {"ok" => true, "user" => {"profile" => {"real_name" => user1.name}}} + ) + stub_slack_users_info_request( + token: token, + user_id: user2.id, + body: {"ok" => true, "user" => {"profile" => {"real_name" => user2.name}}} + ) + slack = described_class.build(api_token: token).value! + + messages = slack.search_messages(search_query).messages.matches + + expect(messages).to match_array [ + msg1.merge("author_real_name" => user1.name), + msg2.merge("author_real_name" => user2.name) + ] + end + + it "searches author names only once per user" do + token = "valid-token" + stub_slack_auth_test_request(status: 200, token: token) + search_query = "tip in:dev" + user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") + msg1 = {"id" => "msg1", "text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"} + msg2 = {"id" => "msg2", "text" => "Ruby tip/TIL: Array#sample...", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-2-permalink.com"} + stub_slack_message_search_request( + query: search_query, + body: { + "ok" => true, + "messages" => { + "matches" => [msg1, msg2], + "paging" => {"pages" => 1} } + } + ) + stub_slack_users_info_request( + token: token, + user_id: user1.id, + body: {"ok" => true, "user" => {"profile" => {"real_name" => user1.name}}} + ) + slack = described_class.build(api_token: token).value! + + slack.search_messages(search_query).messages.matches + + expect(WebMock).to have_requested(:post, "https://slack.com/api/users.info").once + end + + it "searches author names asynchronously" do + search_query_time = 0.5 + sleepy_slack_client = instance_double(Slack::Web::Client, auth_test: true) + sleepy_slack_class = class_double(Slack::Web::Client, new: sleepy_slack_client) + + search_query = "tip in:dev" + user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") + user2 = TestFactories.create_slack_user(id: "user-id-2", name: "User 2", username: "username-2") + msg1 = {"id" => "msg1", "text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"} + msg2 = {"id" => "msg2", "text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"} + allow(sleepy_slack_client).to receive(:search_messages).with({query: search_query}) { + sleep(search_query_time) + + deep_open_struct( + {"ok" => true, + "messages" => { + "matches" => [msg1, msg2], + "paging" => {"pages" => 1} + }} ) - stub_slack_users_info_request( - token: token, - user_id: user1.id, - body: {"ok" => true, "user" => {"profile" => {"real_name" => user1.name}}} - ) - stub_slack_users_info_request( - token: token, - user_id: user2.id, - body: {"ok" => true, "user" => {"profile" => {"real_name" => user2.name}}} - ) - stub_slack_users_info_request( - token: token, - user_id: user3.id, - body: {"ok" => true, "user" => {"profile" => {"real_name" => user3.name}}} - ) - stub_slack_users_info_request( - token: token, - user_id: "user4-id", - body: {"ok" => true, "user" => {"profile" => {"real_name" => "User 4"}}} - ) - author_config = GoldMiner::AuthorConfig.new({ - user1.username => {"link" => user1.link}, - user2.username => {"link" => user2.link}, - user3.username => {"link" => user3.link} - }) - slack = GoldMiner::Slack::Client.build(api_token: token, author_config:).value! - - messages = slack.search_interesting_messages_in("dev") - - expect(messages).to match_array [ - GoldMiner::Slack::Message.new(text: "TIL", author: user1, permalink: "https:///message-1-permalink.com"), - GoldMiner::Slack::Message.new(text: "Ruby tip/TIL: Array#sample...", author: user2, permalink: "https:///message-2-permalink.com"), - GoldMiner::Slack::Message.new(text: "Ruby tip: have fun!", author: user2, permalink: "https:///message-3-permalink.com"), - GoldMiner::Slack::Message.new(text: "CSS clamp() is so cool!", author: user3, permalink: "https:///message-4-permalink.com") - ] - end + } + + user_info_query_time = 1 + allow(sleepy_slack_client).to receive(:users_info).with({user: user1.id}) { + sleep(user_info_query_time) + deep_open_struct({"ok" => true, "user" => {"profile" => {"real_name" => user1.name}}}) + } + allow(sleepy_slack_client).to receive(:users_info).with({user: user2.id}) { + sleep(user_info_query_time) + deep_open_struct({"ok" => true, "user" => {"profile" => {"real_name" => user2.name}}}) + } + + slack = described_class.build(api_token: "valid-token", slack_client: sleepy_slack_class).value! + + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + slack.search_messages(search_query) + total_elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0 + + # We have to sum the search and user info query times because they run + # sequentially. Both user info queries run in parallel, so we don't have + # to sum them twice. + overhead = 0.1 + expect(total_elapsed_time).to be_within(overhead).of(search_query_time + user_info_query_time) end it "warns when results have multiple pages" do travel_to "2022-10-07" do token = "valid-token" + stub_slack_auth_test_request(status: 200, token: token) + search_query = "tip in:dev" user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") user2 = TestFactories.create_slack_user(id: "user-id-2", name: "User 2", username: "username-2") - stub_slack_auth_test_request(status: 200, token: token) + msg1 = {"id" => "msg1", "text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"} + msg2 = {"id" => "msg2", "text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"} stub_slack_message_search_request( - query: "TIL in:dev after:2022-09-30", + query: search_query, body: { "ok" => true, "messages" => { - "matches" => [ - {"text" => "TIL", "user" => user1.id, "username" => user1.username, "permalink" => "https:///message-1-permalink.com"}, - {"text" => "Ruby tip/TIL: Array#sample...", "user" => user2.id, "username" => user2.username, "permalink" => "https:///message-2-permalink.com"} - ], + "matches" => [msg1, msg2], "paging" => {"pages" => 2} } } ) - stub_slack_message_search_request( - query: "tip in:dev after:2022-09-30", - body: { - "ok" => true, - "messages" => { - "matches" => [], - "paging" => {"pages" => 1} - } - } - ) - stub_slack_message_search_request( - query: "in:dev after:2022-09-30 has::rupee-gold:", - body: { - "ok" => true, - "messages" => { - "matches" => [], - "paging" => {"pages" => 1} - } - } - ) stub_slack_users_info_request( token: token, user_id: user1.id, @@ -180,38 +187,13 @@ user_id: user2.id, body: {"ok" => true, "user" => {"profile" => {"real_name" => user2.name}}} ) - author_config = GoldMiner::AuthorConfig.new({ - user1.username => {"link" => user1.link} - }) - slack = GoldMiner::Slack::Client.build(api_token: token, author_config: author_config).value! + slack = described_class.build(api_token: token).value! expect { - slack.search_interesting_messages_in("dev") + slack.search_messages(search_query) }.to output("[WARNING] Found more than one page of results, only the first page will be processed\n").to_stderr end end - - it "searches messages asynchronously" do - seconds_of_sleep = 1 - mock_client = double("Sleepy Slack Client") - allow(mock_client).to receive(:auth_test).and_return(status: 200) - allow(mock_client).to receive(:search_messages) { - sleep seconds_of_sleep - - double("Slack Search", messages: double("Slack Messages", matches: [], paging: double("Slack Paging", pages: 1))) - } - mock_client_class = double("Sleepy Slack Client Class", new: mock_client) - - slack = GoldMiner::Slack::Client.build(api_token: "a-token", slack_client: mock_client_class).value! - - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - messages = slack.search_interesting_messages_in("dev") - elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0 - - overhead = 0.5 - expect(elapsed_time).to be_between(seconds_of_sleep, seconds_of_sleep + overhead) - expect(messages).to eq [] - end end private diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index aac369c..e456d7f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,3 +26,9 @@ c.syntax = :expect end end + +def deep_open_struct(hash) + require "ostruct" + + JSON.parse hash.to_json, object_class: OpenStruct +end From 5a2e14403b3713e31fd40a07c16bfd1ac538094a Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Thu, 7 Sep 2023 16:39:57 -0300 Subject: [PATCH 3/7] Introduce `GoldMiner::SlackExplorer` This class is responsible for "exploring" Slack and "mining" interesting messages. Those messages, later become a blog post. Now that we have a simple Slack client, we can put the business logic in the explorer. This also opens the door for more explorers in the future, like a Twitter explorer. --- lib/gold_miner.rb | 17 +++- lib/gold_miner/messages_query.rb | 4 - lib/gold_miner/slack_explorer.rb | 61 +++++++++++++ spec/gold_miner/messages_query_spec.rb | 32 ------- spec/gold_miner/slack_explorer_spec.rb | 116 +++++++++++++++++++++++++ spec/gold_miner_spec.rb | 21 ++++- 6 files changed, 209 insertions(+), 42 deletions(-) create mode 100644 lib/gold_miner/slack_explorer.rb create mode 100644 spec/gold_miner/slack_explorer_spec.rb diff --git a/lib/gold_miner.rb b/lib/gold_miner.rb index f96ed3c..b68051d 100644 --- a/lib/gold_miner.rb +++ b/lib/gold_miner.rb @@ -12,9 +12,8 @@ module GoldMiner def mine_in(slack_channel, slack_client: GoldMiner::Slack::Client, env_file: ".env") Dotenv.load!(env_file) - slack_client - .build(api_token: ENV["SLACK_API_TOKEN"]) - .fmap { |client| client.search_interesting_messages_in(slack_channel) } + prepare(slack_client) + .fmap { |client| explore(slack_channel, client) } end def convert_messages_to_blogpost(channel, messages, blog_post_builder: GoldMiner::BlogPost) @@ -25,4 +24,16 @@ def convert_messages_to_blogpost(channel, messages, blog_post_builder: GoldMiner writer: BlogPost::Writer.from_env ) end + + private + + def prepare(slack_client) + slack_client.build(api_token: ENV["SLACK_API_TOKEN"]) + end + + def explore(slack_channel, slack_client) + SlackExplorer + .new(slack_client, AuthorConfig.default) + .explore(slack_channel, start_on: Helpers::Time.last_friday) + end end diff --git a/lib/gold_miner/messages_query.rb b/lib/gold_miner/messages_query.rb index 301c006..fc55b9c 100644 --- a/lib/gold_miner/messages_query.rb +++ b/lib/gold_miner/messages_query.rb @@ -17,10 +17,6 @@ def sent_after(new_start_date) with(start_date: new_start_date) end - def sent_after_last_friday - sent_after(Helpers::Time.last_friday) - end - def with_topic(new_topic) with(topic: new_topic) end diff --git a/lib/gold_miner/slack_explorer.rb b/lib/gold_miner/slack_explorer.rb new file mode 100644 index 0000000..875863b --- /dev/null +++ b/lib/gold_miner/slack_explorer.rb @@ -0,0 +1,61 @@ +require "async" + +module GoldMiner + class SlackExplorer + def initialize(slack_client, author_config) + @slack = slack_client + @author_config = author_config + end + + def explore(channel, start_on:) + interesting_messages = interesting_messages_query(channel, start_on) + + Sync do + til_messages_task = Async { search_messages(interesting_messages.with_topic("TIL")) } + tip_messages_task = Async { search_messages(interesting_messages.with_topic("tip")) } + hand_picked_messages_task = Async { search_messages(interesting_messages.with_reaction("rupee-gold")) } + + merge_messages(til_messages_task, tip_messages_task, hand_picked_messages_task) + end + end + + private + + def interesting_messages_query(channel, start_on) + MessagesQuery + .new + .on_channel(channel) + .sent_after(Date.parse(start_on.to_s)) + end + + def merge_messages(*search_tasks) + search_tasks + .flat_map { |task| task.wait.messages.matches } + .uniq { |message| message[:permalink] } + .map { |message| + Slack::Message.new( + text: message.text, + author: author_of(message), + permalink: message.permalink + ) + } + end + + def author_of(message) + Slack::User.new( + name: message.author_real_name, + link: link_for(message.username), + id: message.user, + username: message.username + ) + end + + def search_messages(query) + @slack.search_messages(query.to_s) + end + + def link_for(username) + @author_config.link_for(username) + end + end +end diff --git a/spec/gold_miner/messages_query_spec.rb b/spec/gold_miner/messages_query_spec.rb index e0f4310..7431eac 100644 --- a/spec/gold_miner/messages_query_spec.rb +++ b/spec/gold_miner/messages_query_spec.rb @@ -43,38 +43,6 @@ end end - describe "#sent_after_last_friday" do - it "sets the start date to the last Friday", :aggregate_failures do - a_thursday = "2022-10-06" - a_friday = "2022-10-07" - a_saturday = "2022-10-08" - - travel_to a_thursday do - query = GoldMiner::MessagesQuery.new - - result = query.sent_after_last_friday - - expect(result.start_date).to eq("2022-09-30") - end - - travel_to a_friday do - query = GoldMiner::MessagesQuery.new - - result = query.sent_after_last_friday - - expect(result.start_date).to eq("2022-09-30") # a week before, not today - end - - travel_to a_saturday do - query = GoldMiner::MessagesQuery.new - - result = query.sent_after_last_friday - - expect(result.start_date).to eq("2022-10-07") # the day before - end - end - end - describe "#to_s" do it "returns the string representation of the query" do query = GoldMiner::MessagesQuery.new diff --git a/spec/gold_miner/slack_explorer_spec.rb b/spec/gold_miner/slack_explorer_spec.rb new file mode 100644 index 0000000..a873a6f --- /dev/null +++ b/spec/gold_miner/slack_explorer_spec.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe GoldMiner::SlackExplorer do + describe "#explore" do + it "returns uniq interesting Slack messages sent on dev channel since last friday" do + travel_to "2022-10-07" do + user1 = TestFactories.create_slack_user(id: "user-id-1", name: "User 1", username: "username-1") + user2 = TestFactories.create_slack_user(id: "user-id-2", name: "User 2", username: "username-2") + user3 = TestFactories.create_slack_user(id: "user-id-3", name: "User 3", username: "username-3") + + author_config = GoldMiner::AuthorConfig.new({ + user1.username => {"link" => user1.link}, + user2.username => {"link" => user2.link}, + user3.username => {"link" => user3.link} + }) + slack_client = instance_double(GoldMiner::Slack::Client) + date = "2022-09-30" + stub_slack_message_search_requests(slack_client, { + "TIL in:dev after:#{date}" => { + "ok" => true, + "messages" => { + "matches" => [ + {"text" => "TIL", + "user" => user1.id, + "username" => user1.username, + "author_real_name" => user1.name, + "permalink" => "https:///message-1-permalink.com"}, + {"text" => "Ruby tip/TIL: Array#sample...", + "user" => user2.id, + "username" => user2.username, + "author_real_name" => user2.name, + "permalink" => "https:///message-2-permalink.com"} + ], + "paging" => {"pages" => 1} + } + }, + "tip in:dev after:#{date}" => { + "ok" => true, + "messages" => { + "matches" => [ + {"text" => "Ruby tip/TIL: Array#sample...", + "user" => user2.id, + "username" => user2.username, + "author_real_name" => user2.name, + "permalink" => "https:///message-2-permalink.com"}, + {"text" => "Ruby tip: have fun!", + "user" => user2.id, + "username" => user2.username, + "author_real_name" => user2.name, + "permalink" => "https:///message-3-permalink.com"} + ], + "paging" => {"pages" => 1} + } + }, + "in:dev after:#{date} has::rupee-gold:" => { + "messages" => { + "matches" => [ + {"text" => "Ruby tip/TIL: Array#sample...", + "user" => user2.id, + "username" => user2.username, + "author_real_name" => user2.name, + "permalink" => "https:///message-2-permalink.com"}, + {"text" => "CSS clamp() is so cool!", + "user" => user3.id, + "username" => user3.username, + "author_real_name" => user3.name, + "permalink" => "https:///message-4-permalink.com"} + ], + "paging" => {"pages" => 1} + } + } + }) + + explorer = described_class.new(slack_client, author_config) + messages = explorer.explore("dev", start_on: date) + + expect(messages).to match_array [ + GoldMiner::Slack::Message.new(text: "TIL", author: user1, permalink: "https:///message-1-permalink.com"), + GoldMiner::Slack::Message.new(text: "Ruby tip/TIL: Array#sample...", author: user2, permalink: "https:///message-2-permalink.com"), + GoldMiner::Slack::Message.new(text: "Ruby tip: have fun!", author: user2, permalink: "https:///message-3-permalink.com"), + GoldMiner::Slack::Message.new(text: "CSS clamp() is so cool!", author: user3, permalink: "https:///message-4-permalink.com") + ] + end + end + + it "searches Slack messages asynchronously" do + seconds_of_sleep = 0.5 + mock_client = double("Sleepy Slack Client") + allow(mock_client).to receive(:auth_test).and_return(status: 200) + allow(mock_client).to receive(:search_messages) { + sleep seconds_of_sleep + + double("Search result", messages: double("Slack messages", matches: [], paging: double("Slack paging", pages: 1))) + } + slack_explorer = described_class.new(mock_client, GoldMiner::AuthorConfig.new({})) + + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + slack_explorer.explore("dev", start_on: Date.parse("2022-09-30")) + elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0 + + overhead = 0.1 + expect(elapsed_time).to be_within(overhead).of(seconds_of_sleep) + expect(mock_client).to have_received(:search_messages).thrice + end + + private + + def stub_slack_message_search_requests(client, requests) + requests.map do |query, response| + allow(client).to receive(:search_messages).with(query).and_return(deep_open_struct(response)) + end + end + end +end diff --git a/spec/gold_miner_spec.rb b/spec/gold_miner_spec.rb index 7e1fdab..0b82c03 100644 --- a/spec/gold_miner_spec.rb +++ b/spec/gold_miner_spec.rb @@ -15,13 +15,28 @@ end it "returns interesting messages from the given channel" do - messages = {text: "text", author_username: "user1", permalink: "http://permalink-1.com"} - slack_client = instance_double(GoldMiner::Slack::Client, search_interesting_messages_in: messages) + message_author = TestFactories.create_slack_user(link: "#to-do") + slack_message = TestFactories.create_slack_message(author: message_author) + search_result = deep_open_struct({ + messages: { + matches: [ + { + text: slack_message.text, + user: message_author.id, + username: message_author.username, + author_real_name: message_author.name, + permalink: slack_message.permalink + } + ] + } + }) + + slack_client = instance_double(GoldMiner::Slack::Client, search_messages: search_result) slack_client_builder = double(GoldMiner::Slack::Client, build: Success(slack_client)) result = GoldMiner.mine_in("dev", slack_client: slack_client_builder, env_file: "./spec/fixtures/.env.test") - expect(result.value!).to eq messages + expect(result.value!).to eq [slack_message] end end From d094c04c4fb8f1ebb13a59ad3bb7838f78a67b4c Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Fri, 8 Sep 2023 12:11:05 -0300 Subject: [PATCH 4/7] Remove unused const --- lib/gold_miner/slack/client.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/gold_miner/slack/client.rb b/lib/gold_miner/slack/client.rb index 787d2a9..64b9166 100644 --- a/lib/gold_miner/slack/client.rb +++ b/lib/gold_miner/slack/client.rb @@ -5,8 +5,6 @@ module GoldMiner class Slack::Client - GOLD_EMOJI = "rupee-gold" - extend Dry::Monads[:result] def self.build(api_token:, slack_client: ::Slack::Web::Client) From 3ce66e8886d8d32d276c770d9a2173c44daca2bc Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Fri, 8 Sep 2023 12:11:45 -0300 Subject: [PATCH 5/7] Remove unnecessary double-caching --- lib/gold_miner/slack/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gold_miner/slack/client.rb b/lib/gold_miner/slack/client.rb index 64b9166..b3d55dc 100644 --- a/lib/gold_miner/slack/client.rb +++ b/lib/gold_miner/slack/client.rb @@ -71,7 +71,7 @@ def real_name_for(user_id) .user .profile .real_name - .tap { |name| @user_name_cache[user_id] = name } + # .tap { |name| @user_name_cache[user_id] = name } end def user_info(user_id) From 707178484d1e6f59db3cfe2322a9d02b913de31d Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Fri, 8 Sep 2023 12:13:33 -0300 Subject: [PATCH 6/7] Move GoldMiner::MessagesQuery to GoldMiner::Slack::MessagesQuery --- lib/gold_miner/messages_query.rb | 43 ------------------------ lib/gold_miner/slack/messages_query.rb | 45 ++++++++++++++++++++++++++ lib/gold_miner/slack_explorer.rb | 2 +- spec/gold_miner/messages_query_spec.rb | 14 ++++---- 4 files changed, 53 insertions(+), 51 deletions(-) delete mode 100644 lib/gold_miner/messages_query.rb create mode 100644 lib/gold_miner/slack/messages_query.rb diff --git a/lib/gold_miner/messages_query.rb b/lib/gold_miner/messages_query.rb deleted file mode 100644 index fc55b9c..0000000 --- a/lib/gold_miner/messages_query.rb +++ /dev/null @@ -1,43 +0,0 @@ -module GoldMiner - class MessagesQuery - attr_reader :channel, :start_date, :topic, :reaction - - def initialize(channel: nil, start_date: nil, topic: nil, reaction: nil) - @channel = channel - @start_date = start_date - @topic = topic - @reaction = reaction - end - - def on_channel(new_channel) - with(channel: new_channel) - end - - def sent_after(new_start_date) - with(start_date: new_start_date) - end - - def with_topic(new_topic) - with(topic: new_topic) - end - - def with_reaction(new_reaction) - with(reaction: new_reaction) - end - - def with(**new_attributes) - old_attributes = {channel: channel, start_date: start_date, topic: topic, reaction: reaction} - - self.class.new(**old_attributes.merge(new_attributes)) - end - - def to_s - [ - topic, - channel && "in:#{channel}", - start_date && "after:#{start_date}", - reaction && "has::#{reaction}:" - ].compact.join(" ") - end - end -end diff --git a/lib/gold_miner/slack/messages_query.rb b/lib/gold_miner/slack/messages_query.rb new file mode 100644 index 0000000..e10413b --- /dev/null +++ b/lib/gold_miner/slack/messages_query.rb @@ -0,0 +1,45 @@ +module GoldMiner + module Slack + class MessagesQuery + attr_reader :channel, :start_date, :topic, :reaction + + def initialize(channel: nil, start_date: nil, topic: nil, reaction: nil) + @channel = channel + @start_date = start_date + @topic = topic + @reaction = reaction + end + + def on_channel(new_channel) + with(channel: new_channel) + end + + def sent_after(new_start_date) + with(start_date: new_start_date) + end + + def with_topic(new_topic) + with(topic: new_topic) + end + + def with_reaction(new_reaction) + with(reaction: new_reaction) + end + + def with(**new_attributes) + old_attributes = {channel: channel, start_date: start_date, topic: topic, reaction: reaction} + + self.class.new(**old_attributes.merge(new_attributes)) + end + + def to_s + [ + topic, + channel && "in:#{channel}", + start_date && "after:#{start_date}", + reaction && "has::#{reaction}:" + ].compact.join(" ") + end + end + end +end diff --git a/lib/gold_miner/slack_explorer.rb b/lib/gold_miner/slack_explorer.rb index 875863b..2ce4263 100644 --- a/lib/gold_miner/slack_explorer.rb +++ b/lib/gold_miner/slack_explorer.rb @@ -22,7 +22,7 @@ def explore(channel, start_on:) private def interesting_messages_query(channel, start_on) - MessagesQuery + Slack::MessagesQuery .new .on_channel(channel) .sent_after(Date.parse(start_on.to_s)) diff --git a/spec/gold_miner/messages_query_spec.rb b/spec/gold_miner/messages_query_spec.rb index 7431eac..9e9186a 100644 --- a/spec/gold_miner/messages_query_spec.rb +++ b/spec/gold_miner/messages_query_spec.rb @@ -2,10 +2,10 @@ require "spec_helper" -RSpec.describe GoldMiner::MessagesQuery do +RSpec.describe GoldMiner::Slack::MessagesQuery do describe "#on_channel" do it "sets the channel to search messages in" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query.on_channel("dev") @@ -15,7 +15,7 @@ describe "#sent_after" do it "sets the start date to search messages" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query.sent_after("2018-01-01") @@ -25,7 +25,7 @@ describe "#with_reaction" do it "sets the query reaction to search messages" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query.with_reaction("thumbsup") @@ -35,7 +35,7 @@ describe "#with_topic" do it "sets the query topic to TIL messages" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query.with_topic("TIL") @@ -45,7 +45,7 @@ describe "#to_s" do it "returns the string representation of the query" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query .with_topic("TIL") @@ -58,7 +58,7 @@ end it "does not include unset options" do - query = GoldMiner::MessagesQuery.new + query = described_class.new result = query.to_s From dee6bd93ac07434303d32f727a86ae638d862396 Mon Sep 17 00:00:00 2001 From: Matheus Richard Date: Fri, 8 Sep 2023 12:14:40 -0300 Subject: [PATCH 7/7] Improve spec --- spec/gold_miner/slack/client_spec.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/gold_miner/slack/client_spec.rb b/spec/gold_miner/slack/client_spec.rb index d01b7ec..6bf2fc1 100644 --- a/spec/gold_miner/slack/client_spec.rb +++ b/spec/gold_miner/slack/client_spec.rb @@ -156,6 +156,9 @@ # to sum them twice. overhead = 0.1 expect(total_elapsed_time).to be_within(overhead).of(search_query_time + user_info_query_time) + expect(sleepy_slack_client).to have_received(:search_messages).with({query: search_query}).once + expect(sleepy_slack_client).to have_received(:users_info).with({user: user1.id}).once + expect(sleepy_slack_client).to have_received(:users_info).with({user: user2.id}).once end it "warns when results have multiple pages" do