diff --git a/CHANGES.md b/CHANGES.md index 38adef9..b36b73c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +netuitive_ruby_api 1.1.0 (2016-10-21) +------------------------- +* caching samples/events for the purpose of batching calls to the daemon +* error proofing + netuitive_ruby_api 1.0.1 (2016-10-17) ------------------------- * refactoring of classes into gem namespace to avoid collisions diff --git a/config/agent.yml b/config/agent.yml index 9ab3f69..481faae 100644 --- a/config/agent.yml +++ b/config/agent.yml @@ -1,7 +1,19 @@ #all are configurable using environment variables -logLocation: #absolute path of log file. leave blank for default location in the gem directory. environment variable: NETUITIVE_RUBY_LOG_LOCATION -logAge: daily #Number of old log files to keep, or frequency of rotation (daily, weekly or monthly). environment variable: NETUITIVE_RUBY_LOG_AGE -logSize: #Maximum logfile size in bytes (only applies when shift_age is a number). environment variable: NETUITIVE_RUBY_LOG_SIZE -debugLevel: info #options (in ascending order of debugness) are: error, info, debug. environment variable: NETUITIVE_RUBY_DEBUG_LEVEL -netuitivedAddr: localhost #environment variable: NETUITIVE_RUBY_NETUITIVED_ADDR -netuitivedPort: 8875 #environment variable: NETUITIVE_RUBY_NETUITIVED_PORT +#log properties +logLocation: #NETUITIVE_RUBY_LOG_LOCATION absolute path of log file. leave blank for default location in the gem directory. +logAge: daily #NETUITIVE_RUBY_LOG_AGE Number of old log files to keep, or frequency of rotation (daily, weekly or monthly). +logSize: #NETUITIVE_RUBY_LOG_SIZE Maximum logfile size in bytes (only applies when shift_age is a number). +debugLevel: error #NETUITIVE_RUBY_DEBUG_LEVEL options (in ascending order of debugness) are: error, info, debug. + +#netuitived connection properties +netuitivedAddr: localhost #NETUITIVE_RUBY_NETUITIVED_ADDR +netuitivedPort: 8875 #NETUITIVE_RUBY_NETUITIVED_PORT + +#cache properties +#the point of the cache is to be as *small* as possible while still avoiding excessive delivery thread growth. +sampleCacheEnabled: true #NETUITIVE_RUBY_SAMPLE_CACHE_ENABLED +sampleCacheSize: 50 #NETUITIVE_RUBY_SAMPLE_CACHE_SIZE maximum number of samples to be cached before being sent to netuitived +sampleCacheInterval: 10 #NETUITIVE_RUBY_SAMPLE_CACHE_INTERVAL interval (in seconds) to send cached samples to netuitived +eventCacheEnabled: false #NETUITIVE_RUBY_EVENT_CACHE_ENABLED +eventCacheSize: 50 #NETUITIVE_RUBY_SAMPLE_CACHE_SIZE maximum number of events to be cached before being sent to netuitived +eventCacheInterval: 10 #NETUITIVE_RUBY_SAMPLE_CACHE_INTERVAL interval (in seconds) to send cached events to netuitived diff --git a/lib/netuitive_ruby_api.rb b/lib/netuitive_ruby_api.rb index da3b74e..61c644c 100644 --- a/lib/netuitive_ruby_api.rb +++ b/lib/netuitive_ruby_api.rb @@ -3,39 +3,81 @@ require 'drb/drb' require 'netuitive_ruby_api/config_manager' require 'netuitive_ruby_api/netuitive_logger' +require 'netuitive_ruby_api/error_logger' +require 'netuitive_ruby_api/data_cache' +require 'netuitive_ruby_api/data_manager' +require 'netuitive_ruby_api/event_schedule' +require 'netuitive_ruby_api/sample_schedule' class NetuitiveRubyAPI class << self - def setup(server) - @@netuitivedServer = server + attr_accessor :data_manager + attr_accessor :netuitivedServer + attr_reader :pid + def setup + @pid = Process.pid + NetuitiveRubyApi::ConfigManager.load_config + NetuitiveRubyApi::NetuitiveLogger.setup + NetuitiveRubyApi::ConfigManager.read_config + NetuitiveRubyApi::ErrorLogger.guard('error during api setup') do + server_uri = "druby://#{NetuitiveRubyApi::ConfigManager.netuitivedAddr}:#{NetuitiveRubyApi::ConfigManager.netuitivedPort}".freeze + DRb.start_service + drb_server = DRbObject.new_with_uri(server_uri) + data_manager = NetuitiveRubyApi::DataManager.new + data_manager.data_cache = NetuitiveRubyApi::DataCache.new + data_manager.sample_cache_enabled = NetuitiveRubyApi::ConfigManager.sample_cache_enabled + data_manager.sample_cache_size = NetuitiveRubyApi::ConfigManager.sample_cache_size + data_manager.sample_cache_interval = NetuitiveRubyApi::ConfigManager.sample_cache_interval + data_manager.event_cache_enabled = NetuitiveRubyApi::ConfigManager.event_cache_enabled + data_manager.event_cache_size = NetuitiveRubyApi::ConfigManager.event_cache_size + data_manager.event_cache_interval = NetuitiveRubyApi::ConfigManager.event_cache_interval + data_manager.netuitived_server = drb_server + @netuitivedServer = drb_server + @data_manager = data_manager + NetuitiveRubyApi::SampleSchedule.stop + NetuitiveRubyApi::EventSchedule.stop + NetuitiveRubyApi::SampleSchedule.start(NetuitiveRubyApi::ConfigManager.sample_cache_interval) if NetuitiveRubyApi::ConfigManager.sample_cache_enabled + NetuitiveRubyApi::EventSchedule.start(NetuitiveRubyApi::ConfigManager.event_cache_interval) if NetuitiveRubyApi::ConfigManager.event_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.info 'netuitive_ruby_api finished setup' + end + end + + def flush_samples + @data_manager.flush_samples + end + + def check_restart + NetuitiveRubyApi::NetuitiveLogger.log.debug "stored pid: #{@pid}, process pid: #{Process.pid}" + return if @pid == Process.pid + Thread.new { NetuitiveRubyAPI.setup } end - def netuitivedServer - @@netuitivedServer + def flush_events + @data_manager.flush_events end def send_metrics - server_interaction { netuitivedServer.sendMetrics } + netuitivedServer.sendMetrics end def add_sample(metric_id, val) - server_interaction { netuitivedServer.addSample(metric_id, val) } + @data_manager.add_sample(metric_id, val) end def add_counter_sample(metric_id, val) - server_interaction { netuitivedServer.addCounterSample(metric_id, val) } + @data_manager.add_counter_sample(metric_id, val) end def aggregate_metric(metric_id, val) - server_interaction { netuitivedServer.aggregateMetric(metric_id, val) } + @data_manager.aggregate_metric(metric_id, val) end def aggregate_counter_metric(metric_id, val) - server_interaction { netuitivedServer.aggregateCounterMetric(metric_id, val) } + @data_manager.aggregate_counter_metric(metric_id, val) end def clear_metrics - server_interaction { netuitivedServer.clearMetrics } + netuitivedServer.clearMetrics end def interval @@ -43,36 +85,17 @@ def interval end def event(message, timestamp = Time.new, title = 'Ruby Event', level = 'Info', source = 'Ruby Agent', type = 'INFO', tags = nil) - server_interaction { netuitivedServer.event(message, timestamp, title, level, source, type, tags) } + @data_manager.event(message, timestamp, title, level, source, type, tags) end def exception_event(exception, klass = nil, tags = nil) - server_interaction do - hash = { message: exception.message } - hash[:backtrace] = exception.backtrace.join("\n\t") if (defined? exception.backtrace) && !exception.backtrace.nil? - netuitivedServer.exceptionEvent(hash, klass, tags) - end + @data_manager.exception_event(exception, klass, tags) end def stop_server - server_interaction { netuitivedServer.stopServer } - end - - def server_interaction - Thread.new do - begin - yield - rescue => e - NetuitiveRubyApi::NetuitiveLogger.log.error "unable to connect to netuitived: message:#{e.message} backtrace:#{e.backtrace}" - end - end + netuitivedServer.stopServer end end end -NetuitiveRubyApi::ConfigManager.load_config -NetuitiveRubyApi::NetuitiveLogger.setup -NetuitiveRubyApi::ConfigManager.read_config -SERVER_URI = "druby://#{NetuitiveRubyApi::ConfigManager.netuitivedAddr}:#{NetuitiveRubyApi::ConfigManager.netuitivedPort}".freeze -DRb.start_service -NetuitiveRubyAPI.setup(DRbObject.new_with_uri(SERVER_URI)) +NetuitiveRubyAPI.setup diff --git a/lib/netuitive_ruby_api/config_manager.rb b/lib/netuitive_ruby_api/config_manager.rb index 9b0191e..96cd05a 100644 --- a/lib/netuitive_ruby_api/config_manager.rb +++ b/lib/netuitive_ruby_api/config_manager.rb @@ -2,10 +2,14 @@ module NetuitiveRubyApi class ConfigManager class << self attr_reader :netuitivedAddr - attr_reader :netuitivedPort - attr_reader :data + attr_reader :sample_cache_enabled + attr_reader :sample_cache_size + attr_reader :sample_cache_interval + attr_reader :event_cache_enabled + attr_reader :event_cache_size + attr_reader :event_cache_interval def property(name, var, default = nil) prop = ENV[var] @@ -26,12 +30,11 @@ def boolean_property(name, var) end def float_property(name, var) - prop = ENV[var].nil? ? nil : ENV[var] - if prop.nil? || (prop == '') - data[name].to_f - else - prop.to_f - end + property(name, var).to_f + end + + def int_property(name, var) + property(name, var).to_i end def string_list_property(name, var) @@ -52,6 +55,12 @@ def load_config end def read_config + @sample_cache_enabled = boolean_property('sampleCacheEnabled', 'NETUITIVE_RUBY_SAMPLE_CACHE_ENABLED') + @sample_cache_size = int_property('sampleCacheSize', 'NETUITIVE_RUBY_SAMPLE_CACHE_SIZE') + @sample_cache_interval = int_property('sampleCacheInterval', 'NETUITIVE_RUBY_SAMPLE_CACHE_INTERVAL') + @event_cache_enabled = boolean_property('eventCacheEnabled', 'NETUITIVE_RUBY_EVENT_CACHE_ENABLED') + @event_cache_size = int_property('eventCacheSize', 'NETUITIVE_RUBY_SAMPLE_CACHE_SIZE') + @event_cache_interval = int_property('eventCacheInterval', 'NETUITIVE_RUBY_SAMPLE_CACHE_INTERVAL') @netuitivedAddr = property('netuitivedAddr', 'NETUITIVE_RUBY_NETUITIVED_ADDR') @netuitivedPort = property('netuitivedPort', 'NETUITIVE_RUBY_NETUITIVED_PORT') debugLevelString = property('debugLevel', 'NETUITIVE_RUBY_DEBUG_LEVEL') @@ -64,12 +73,18 @@ def read_config else Logger::ERROR end - NetuitiveRubyApi::NetuitiveLogger.log.info "port: #{@netuitivedPort}" - NetuitiveRubyApi::NetuitiveLogger.log.info "addr: #{@netuitivedAddr}" + NetuitiveRubyApi::NetuitiveLogger.log.info "netuitived port: #{@netuitivedPort}" + NetuitiveRubyApi::NetuitiveLogger.log.info "netuitived addr: #{@netuitivedAddr}" NetuitiveRubyApi::NetuitiveLogger.log.debug "read config file. Results: netuitivedAddr: #{@netuitivedAddr} netuitivedPort: #{@netuitivedPort} - debugLevel: #{debugLevelString}" + debugLevel: #{debugLevelString} + sample_cache_enabled: #{@sample_cache_enabled} + sample_cache_size: #{@sample_cache_size} + sample_cache_interval: #{@sample_cache_interval} + event_cache_enabled: #{@event_cache_enabled} + event_cache_size: #{@event_cache_size} + event_cache_interval: #{@event_cache_interval}" end end end diff --git a/lib/netuitive_ruby_api/data_cache.rb b/lib/netuitive_ruby_api/data_cache.rb new file mode 100644 index 0000000..628f6b5 --- /dev/null +++ b/lib/netuitive_ruby_api/data_cache.rb @@ -0,0 +1,104 @@ +module NetuitiveRubyApi + class DataCache + def initialize + @sample_count_mutex = Mutex.new + @event_count_mutex = Mutex.new + reset_samples + reset_events + end + + def sample_added + @sample_count += 1 + end + + def event_added + @event_count += 1 + end + + def add_sample(value) + @sample_count_mutex.synchronize do + @samples.push(value) + sample_added + end + end + + def add_counter_sample(value) + @sample_count_mutex.synchronize do + @counter_samples.push(value) + sample_added + end + end + + def add_aggregate_metric(value) + @sample_count_mutex.synchronize do + @aggregate_metrics.push(value) + sample_added + end + end + + def add_aggregate_counter_metric(value) + @sample_count_mutex.synchronize do + @aggregate_counter_metrics.push(value) + sample_added + end + end + + def add_event(value) + @event_count_mutex.synchronize do + @events.push(value) + event_added + end + end + + def add_exception_event(value) + @event_count_mutex.synchronize do + @exception_events.push(value) + event_added + end + end + + def clear_sample_cache + NetuitiveRubyApi::ErrorLogger.guard('error during clear_sample_cache') do + @sample_count_mutex.synchronize do + NetuitiveRubyApi::NetuitiveLogger.log.debug 'clearing sample cache' + ret = { + samples: @samples.dup, + counter_samples: @counter_samples.dup, + aggregate_metrics: @aggregate_metrics.dup, + aggregate_counter_metrics: @aggregate_counter_metrics.dup + } + reset_samples + ret + end + end + end + + def clear_event_cache + NetuitiveRubyApi::ErrorLogger.guard('error during clear_event_cache') do + @event_count_mutex.synchronize do + NetuitiveRubyApi::NetuitiveLogger.log.debug 'clearing event cache' + ret = { + events: @events.dup, + exception_events: @exception_events.dup + } + reset_events + ret + end + end + end + + def reset_samples + @samples = [] + @counter_samples = [] + @aggregate_metrics = [] + @aggregate_counter_metrics = [] + @sample_count = 0 + end + + def reset_events + @events = [] + @exception_events = [] + @event_count = 0 + end + end +end diff --git a/lib/netuitive_ruby_api/data_manager.rb b/lib/netuitive_ruby_api/data_manager.rb new file mode 100644 index 0000000..3368c9c --- /dev/null +++ b/lib/netuitive_ruby_api/data_manager.rb @@ -0,0 +1,142 @@ +module NetuitiveRubyApi + class DataManager + attr_accessor :data_cache + attr_accessor :sample_cache_enabled + attr_accessor :sample_cache_size + attr_accessor :sample_cache_interval + attr_accessor :event_cache_enabled + attr_accessor :event_cache_size + attr_accessor :event_cache_interval + attr_accessor :netuitived_server + + def add_sample(metric_id, val) + if @sample_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding sample to cache: #{metric_id}" + sample_cache_addition { data_cache.add_sample(metric_id: metric_id, val: val) } + else + server_interaction { netuitived_server.addSample(metric_id, val) } + end + end + + def add_counter_sample(metric_id, val) + if @sample_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding sample to cache: #{metric_id}" + sample_cache_addition { data_cache.add_counter_sample(metric_id: metric_id, val: val) } + else + server_interaction { netuitived_server.addCounterSample(metric_id, val) } + end + end + + def aggregate_metric(metric_id, val) + if @sample_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding sample to cache: #{metric_id}" + sample_cache_addition { data_cache.add_aggregate_metric(metric_id: metric_id, val: val) } + else + server_interaction { netuitived_server.aggregateMetric(metric_id, val) } + end + end + + def aggregate_counter_metric(metric_id, val) + if @sample_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding sample to cache: #{metric_id}" + sample_cache_addition { data_cache.add_aggregate_counter_metric(metric_id: metric_id, val: val) } + else + server_interaction { netuitived_server.aggregateCounterMetric(metric_id, val) } + end + end + + def event(message, timestamp, title, level, source, type, tags) + if @event_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding event to cache: #{message}" + event_cache_addition do + data_cache.add_event(message: message, + timestamp: timestamp, + title: title, + level: level, + source: source, + type: type, + tags: tags) + end + else + server_interaction { netuitived_server.event(message, timestamp, title, level, source, type, tags) } + end + end + + def exception_event(exception, klass, tags) + NetuitiveRubyApi::ErrorLogger.guard('error during exception_event') do + hash = { message: exception.message } + hash[:backtrace] = exception.backtrace.join("\n\t") if (defined? exception.backtrace) && !exception.backtrace.nil? + if @event_cache_enabled + NetuitiveRubyApi::NetuitiveLogger.log.debug "adding exception event to cache: #{hash[:message]}" + event_cache_addition do + data_cache.add_exception_event(exception: hash, + klass: klass, + tags: tags) + end + else + server_interaction { netuitived_server.exceptionEvent(hash, klass, tags) } + end + end + end + + def flush_samples + NetuitiveRubyApi::ErrorLogger.guard('error during flush_samples') do + sample = data_cache.clear_sample_cache + num = sample[:samples].size + sample[:counter_samples].size + sample[:aggregate_metrics].size + sample[:aggregate_counter_metrics].size + NetuitiveRubyApi::NetuitiveLogger.log.info "sending #{num} samples" if num > 0 + threads = [] + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending samples: #{sample[:samples]} " + netuitived_server.add_samples sample[:samples] + end unless sample[:samples].empty? + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending counter_samples: #{sample[:counter_samples]} " + netuitived_server.add_counter_samples sample[:counter_samples] + end unless sample[:counter_samples].empty? + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending aggregate_metrics: #{sample[:aggregate_metrics]} " + netuitived_server.add_aggregate_metrics sample[:aggregate_metrics] + end unless sample[:aggregate_metrics].empty? + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending aggregate_counter_metrics: #{sample[:aggregate_counter_metrics]} " + netuitived_server.add_aggregate_counter_metrics sample[:aggregate_counter_metrics] + end unless sample[:aggregate_counter_metrics].empty? + threads + end + end + + def flush_events + NetuitiveRubyApi::ErrorLogger.guard('error during flush_events') do + event_cache = data_cache.clear_event_cache + num = event_cache[:events].size + event_cache[:exception_events].size + NetuitiveRubyApi::NetuitiveLogger.log.info "sending #{num} events" if num > 0 + threads = [] + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending events: #{event_cache[:events]} " + netuitived_server.add_events(event_cache[:events]) + end unless event_cache[:events].empty? + threads << server_interaction do + NetuitiveRubyApi::NetuitiveLogger.log.debug "sending exception_events: #{event_cache[:exception_events]} " + netuitived_server.add_exception_events(event_cache[:exception_events]) + end unless event_cache[:exception_events].empty? + threads + end + end + + def sample_cache_addition + NetuitiveRubyAPI.check_restart + flush_samples if yield >= @sample_cache_size + end + + def event_cache_addition + NetuitiveRubyAPI.check_restart + flush_events if yield >= @event_cache_size + end + + def server_interaction + Thread.new do + NetuitiveRubyApi::ErrorLogger.guard('error during server interaction') { yield } + end + end + end +end diff --git a/lib/netuitive_ruby_api/error_logger.rb b/lib/netuitive_ruby_api/error_logger.rb new file mode 100644 index 0000000..43811a6 --- /dev/null +++ b/lib/netuitive_ruby_api/error_logger.rb @@ -0,0 +1,21 @@ +# Important! ruby rescue logic is expensive. +# This class is *intended* to be used as a catch all, because we don't want any errors to not be logged +# or worse to bubble up to the host application. +# That *doesn't* mean we should be throwing exceptions rather than guarding against them +# From the benchmarks I've read it seems like rescues are free if no exception is thrown +module NetuitiveRubyApi + class ErrorLogger + class << self + def guard(message) + yield + rescue => e + NetuitiveRubyApi::NetuitiveLogger.log.error format_exception(e, message) + end + + def format_exception(exception, *message) + message = '' unless defined? message || message.nil? + "#{message} \n\tException message: #{exception.message}\n\t Backtrace: #{exception.backtrace.join("\n\t")}" + end + end + end +end diff --git a/lib/netuitive_ruby_api/event_schedule.rb b/lib/netuitive_ruby_api/event_schedule.rb new file mode 100644 index 0000000..e023c2c --- /dev/null +++ b/lib/netuitive_ruby_api/event_schedule.rb @@ -0,0 +1,22 @@ +module NetuitiveRubyApi + class EventSchedule + def self.start(interval) + @@thread = Thread.new do + loop do + sleep(interval) + Thread.new do + NetuitiveRubyApi::NetuitiveLogger.log.debug 'started event job' + NetuitiveRubyApi::ErrorLogger.guard('error during event job') do + NetuitiveRubyAPI.flush_events + end + NetuitiveRubyApi::NetuitiveLogger.log.debug 'finished event job' + end + end + end + end + + def self.stop + @@thread.kill if defined? @@thread + end + end +end diff --git a/lib/netuitive_ruby_api/netuitive_logger.rb b/lib/netuitive_ruby_api/netuitive_logger.rb index 7deb19b..271c387 100644 --- a/lib/netuitive_ruby_api/netuitive_logger.rb +++ b/lib/netuitive_ruby_api/netuitive_logger.rb @@ -14,7 +14,7 @@ def info(message) class NetuitiveLogger class << self - attr_reader :log + attr_accessor :log def setup file = NetuitiveRubyApi::ConfigManager.property('logLocation', 'NETUITIVE_RUBY_LOG_LOCATION', "#{File.expand_path('../../..', __FILE__)}/log/netuitive.log") age = NetuitiveRubyApi::ConfigManager.property('logAge', 'NETUITIVE_RUBY_LOG_AGE', 'daily') diff --git a/lib/netuitive_ruby_api/sample_schedule.rb b/lib/netuitive_ruby_api/sample_schedule.rb new file mode 100644 index 0000000..390984b --- /dev/null +++ b/lib/netuitive_ruby_api/sample_schedule.rb @@ -0,0 +1,22 @@ +module NetuitiveRubyApi + class SampleSchedule + def self.start(interval) + @@thread = Thread.new do + loop do + sleep(interval) + Thread.new do + NetuitiveRubyApi::NetuitiveLogger.log.debug 'started sample job' + NetuitiveRubyApi::ErrorLogger.guard('error during sample job') do + NetuitiveRubyAPI.flush_samples + end + NetuitiveRubyApi::NetuitiveLogger.log.debug 'finished sample job' + end + end + end + end + + def self.stop + @@thread.kill if defined? @@thread + end + end +end diff --git a/netuitive_ruby_api.gemspec b/netuitive_ruby_api.gemspec index 6003b4d..85620c7 100644 --- a/netuitive_ruby_api.gemspec +++ b/netuitive_ruby_api.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'netuitive_ruby_api' - s.version = '1.0.1' - s.date = '2016-10-17' + s.version = '1.1.0' + s.date = '2016-10-21' s.summary = "Interface for Netuitive's metric ingest API" s.description = 'Allows for easy submittion of metrics to Netuitive' s.authors = ['John King'] @@ -12,4 +12,5 @@ Gem::Specification.new do |s| 'http://rubygems.org/gems/netuitive_ruby_api' s.license = 'Apache v2.0' s.required_ruby_version = '>= 1.9.0' + s.add_development_dependency 'netuitived', '>= 1.1.0' end diff --git a/test/test_data_cache.rb b/test/test_data_cache.rb new file mode 100644 index 0000000..4dff9bd --- /dev/null +++ b/test/test_data_cache.rb @@ -0,0 +1,128 @@ +require 'test/unit' +require 'mocha/test_unit' +require 'netuitive_ruby_api' +require 'netuitive_ruby_api/config_manager' +require 'netuitive_ruby_api/netuitive_logger' +require 'netuitive_ruby_api/error_logger' +require 'netuitive_ruby_api/data_cache' +require 'netuitive_ruby_api/data_manager' +require 'netuitive_ruby_api/event_schedule' +require 'netuitive_ruby_api/sample_schedule' + +module NetuitiveRubyApi + class DataCacheTest < Test::Unit::TestCase + def setup + @data_cache = NetuitiveRubyApi::DataCache.new + end + + def test_add_sample + count = @data_cache.add_sample(sample) + assert_equal(count, 1) + samples = @data_cache.clear_sample_cache + assert_equal(samples[:samples], [sample]) + assert_equal(samples[:counter_samples], []) + assert_equal(samples[:aggregate_metrics], []) + assert_equal(samples[:aggregate_counter_metrics], []) + end + + def test_add_counter_sample + count = @data_cache.add_counter_sample(sample) + assert_equal(count, 1) + samples = @data_cache.clear_sample_cache + assert_equal(samples[:samples], []) + assert_equal(samples[:counter_samples], [sample]) + assert_equal(samples[:aggregate_metrics], []) + assert_equal(samples[:aggregate_counter_metrics], []) + end + + def test_add_aggregate_metric + count = @data_cache.add_aggregate_metric(sample) + assert_equal(count, 1) + samples = @data_cache.clear_sample_cache + assert_equal(samples[:samples], []) + assert_equal(samples[:counter_samples], []) + assert_equal(samples[:aggregate_metrics], [sample]) + assert_equal(samples[:aggregate_counter_metrics], []) + end + + def test_add_aggregate_counter_metric + count = @data_cache.add_aggregate_counter_metric(sample) + assert_equal(count, 1) + samples = @data_cache.clear_sample_cache + assert_equal(samples[:samples], []) + assert_equal(samples[:counter_samples], []) + assert_equal(samples[:aggregate_metrics], []) + assert_equal(samples[:aggregate_counter_metrics], [sample]) + end + + def test_add_event + count = @data_cache.add_event(event) + assert_equal(count, 1) + events = @data_cache.clear_event_cache + assert_equal(events[:events], [event]) + assert_equal(events[:exception_events], []) + end + + def test_add_exception_event + count = @data_cache.add_exception_event(exception_event) + assert_equal(count, 1) + events = @data_cache.clear_event_cache + assert_equal(events[:events], []) + assert_equal(events[:exception_events], [exception_event]) + end + + def test_sample_locks + threads = [] + start_time = Time.new + 250.times { threads << Thread.new { @data_cache.add_sample(sample) } } + 250.times { threads << Thread.new { @data_cache.add_counter_sample(sample) } } + 250.times { threads << Thread.new { @data_cache.add_aggregate_metric(sample) } } + 250.times { threads << Thread.new { @data_cache.add_aggregate_counter_metric(sample) } } + threads.each(&:join) + end_time = Time.new + assert((end_time - start_time) * 1000 < 500) # we should be able to chew through 1000 samples in 500 ms + count = @data_cache.add_sample(sample) + assert_equal(count, 1001) + samples = @data_cache.clear_sample_cache + assert_equal(samples[:samples].size, 251) + assert_equal(samples[:counter_samples].size, 250) + assert_equal(samples[:aggregate_metrics].size, 250) + assert_equal(samples[:aggregate_counter_metrics].size, 250) + end + + def test_event_locks + threads = [] + start_time = Time.new + 500.times { threads << Thread.new { @data_cache.add_event(event) } } + 500.times { threads << Thread.new { @data_cache.add_exception_event(exception_event) } } + threads.each(&:join) + end_time = Time.new + assert((end_time - start_time) * 1000 < 500) # we should be able to chew through 1000 events in 500 ms + count = @data_cache.add_event(event) + assert_equal(count, 1001) + events = @data_cache.clear_event_cache + assert_equal(events[:events].size, 501) + assert_equal(events[:exception_events].size, 500) + end + + def sample + { metric_id: 'metric.id', val: 1 } + end + + def event + { message: 'test message', + timestamp: Time.new(2000, 1, 1, 1, 1, 1), + title: 'test title', + level: 'test level', + source: 'test source', + type: 'test type', + tags: [{ test_name: 'test value' }] } + end + + def exception_event + { exception: RuntimeError.new, + klass: RuntimeError.class, + tags: [{ test_name: 'test value' }] } + end + end +end diff --git a/test/test_data_manager.rb b/test/test_data_manager.rb new file mode 100644 index 0000000..87205ca --- /dev/null +++ b/test/test_data_manager.rb @@ -0,0 +1,234 @@ +module NetuitiveRubyApi + class DataManagerTest < Test::Unit::TestCase + def setup + @data_cache = mock + @netuitived_server = mock + @data_manager = NetuitiveRubyApi::DataManager.new + @data_manager.data_cache = @data_cache + @data_manager.netuitived_server = @netuitived_server + @error = make_error + end + + def test_add_sample + sample_cache_disable + @netuitived_server.expects(:addSample).once.with('metric.id', 1) + thread = @data_manager.add_sample('metric.id', 1) + thread.join + + sample_cache_enable + sample_cache_size(10) + @data_cache.expects(:add_sample).once.with(sample).returns(1) + @data_manager.add_sample('metric.id', 1) + + sample_cache_enable + sample_cache_size(1) + @data_cache.expects(:add_sample).once.with(sample).returns(1) + @data_cache.expects(:clear_sample_cache).once.returns(sample_cache) + @netuitived_server.expects(:add_samples).once.with(sample_cache[:samples]) + @netuitived_server.expects(:add_counter_samples).once.with(sample_cache[:counter_samples]) + @netuitived_server.expects(:add_aggregate_metrics).once.with(sample_cache[:aggregate_metrics]) + @netuitived_server.expects(:add_aggregate_counter_metrics).once.with(sample_cache[:aggregate_counter_metrics]) + @data_manager.add_sample('metric.id', 1).each(&:join) + end + + def test_add_counter_sample + sample_cache_disable + @netuitived_server.expects(:addCounterSample).once.with('metric.id', 1) + thread = @data_manager.add_counter_sample('metric.id', 1) + thread.join + + sample_cache_enable + sample_cache_size(10) + @data_cache.expects(:add_counter_sample).once.with(sample).returns(1) + @data_manager.add_counter_sample('metric.id', 1) + + sample_cache_enable + sample_cache_size(1) + @data_cache.expects(:add_counter_sample).once.with(sample).returns(1) + @data_cache.expects(:clear_sample_cache).once.returns(sample_cache) + @netuitived_server.expects(:add_samples).once.with(sample_cache[:samples]) + @netuitived_server.expects(:add_counter_samples).once.with(sample_cache[:counter_samples]) + @netuitived_server.expects(:add_aggregate_metrics).once.with(sample_cache[:aggregate_metrics]) + @netuitived_server.expects(:add_aggregate_counter_metrics).once.with(sample_cache[:aggregate_counter_metrics]) + @data_manager.add_counter_sample('metric.id', 1).each(&:join) + end + + def test_aggregate_metric + sample_cache_disable + @netuitived_server.expects(:aggregateMetric).once.with('metric.id', 1) + thread = @data_manager.aggregate_metric('metric.id', 1) + thread.join + + sample_cache_enable + sample_cache_size(10) + @data_cache.expects(:add_aggregate_metric).once.with(sample).returns(1) + @data_manager.aggregate_metric('metric.id', 1) + + sample_cache_enable + sample_cache_size(1) + @data_cache.expects(:add_aggregate_metric).once.with(sample).returns(1) + @data_cache.expects(:clear_sample_cache).once.returns(sample_cache) + @netuitived_server.expects(:add_samples).once.with(sample_cache[:samples]) + @netuitived_server.expects(:add_counter_samples).once.with(sample_cache[:counter_samples]) + @netuitived_server.expects(:add_aggregate_metrics).once.with(sample_cache[:aggregate_metrics]) + @netuitived_server.expects(:add_aggregate_counter_metrics).once.with(sample_cache[:aggregate_counter_metrics]) + @data_manager.aggregate_metric('metric.id', 1).each(&:join) + end + + def test_aggregate_counter_metric + sample_cache_disable + @netuitived_server.expects(:aggregateCounterMetric).once.with('metric.id', 1) + thread = @data_manager.aggregate_counter_metric('metric.id', 1) + thread.join + + sample_cache_enable + sample_cache_size(10) + @data_cache.expects(:add_aggregate_counter_metric).once.with(sample).returns(1) + @data_manager.aggregate_counter_metric('metric.id', 1) + + sample_cache_enable + sample_cache_size(1) + @data_cache.expects(:add_aggregate_counter_metric).once.with(sample).returns(1) + @data_cache.expects(:clear_sample_cache).once.returns(sample_cache) + @netuitived_server.expects(:add_samples).once.with(sample_cache[:samples]) + @netuitived_server.expects(:add_counter_samples).once.with(sample_cache[:counter_samples]) + @netuitived_server.expects(:add_aggregate_metrics).once.with(sample_cache[:aggregate_metrics]) + @netuitived_server.expects(:add_aggregate_counter_metrics).once.with(sample_cache[:aggregate_counter_metrics]) + @data_manager.aggregate_counter_metric('metric.id', 1).each(&:join) + end + + def test_event + event_cache_disable + @netuitived_server.expects(:event).once.with('test message', + Time.new(2000, 1, 1, 1, 1, 1), + 'test title', + 'test level', + 'test source', + 'test type', + [{ test_name: 'test value' }]) + thread = @data_manager.event('test message', + Time.new(2000, 1, 1, 1, 1, 1), + 'test title', + 'test level', + 'test source', + 'test type', + [{ test_name: 'test value' }]) + thread.join + + event_cache_enable + event_cache_size(10) + @data_cache.expects(:add_event).once.with(event).returns(1) + @data_manager.event('test message', + Time.new(2000, 1, 1, 1, 1, 1), + 'test title', + 'test level', + 'test source', + 'test type', + [{ test_name: 'test value' }]) + + event_cache_enable + event_cache_size(1) + @data_cache.expects(:add_event).once.with(event).returns(1) + @data_cache.expects(:clear_event_cache).once.returns(event_cache) + @netuitived_server.expects(:add_events).once.with(event_cache[:events]) + @netuitived_server.expects(:add_exception_events).once.with(event_cache[:exception_events]) + @data_manager.event('test message', + Time.new(2000, 1, 1, 1, 1, 1), + 'test title', + 'test level', + 'test source', + 'test type', + [{ test_name: 'test value' }]).each(&:join) + end + + def test_exception_event + event_cache_disable + @netuitived_server.expects(:exceptionEvent).once.with({ message: @error.message, backtrace: @error.backtrace.join("\n\t") }, + @error.class, + [{ test_name: 'test value' }]) + thread = @data_manager.exception_event(@error, + @error.class, + [{ test_name: 'test value' }]) + thread.join + + event_cache_enable + event_cache_size(10) + @data_cache.expects(:add_exception_event).once.with(exception_event).returns(1) + @data_manager.exception_event(@error, + @error.class, + [{ test_name: 'test value' }]) + + event_cache_enable + event_cache_size(1) + @data_cache.expects(:add_exception_event).once.with(exception_event).returns(1) + @data_cache.expects(:clear_event_cache).once.returns(event_cache) + @netuitived_server.expects(:add_events).once.with(event_cache[:events]) + @netuitived_server.expects(:add_exception_events).once.with(event_cache[:exception_events]) + @data_manager.exception_event(@error, + @error.class, + [{ test_name: 'test value' }]).each(&:join) + end + + def sample_cache_enable + @data_manager.sample_cache_enabled = true + end + + def sample_cache_size(size) + @data_manager.sample_cache_size = size + end + + def sample_cache_disable + @data_manager.sample_cache_enabled = false + end + + def event_cache_enable + @data_manager.event_cache_enabled = true + end + + def event_cache_size(size) + @data_manager.event_cache_size = size + end + + def event_cache_disable + @data_manager.event_cache_enabled = false + end + + def sample + { metric_id: 'metric.id', val: 1 } + end + + def sample_cache + { samples: [sample], + counter_samples: [sample], + aggregate_metrics: [sample], + aggregate_counter_metrics: [sample] } + end + + def event_cache + { events: [event], + exception_events: [exception_event] } + end + + def make_error + raise 'test exception' + rescue => e + return e + end + + def event + { message: 'test message', + timestamp: Time.new(2000, 1, 1, 1, 1, 1), + title: 'test title', + level: 'test level', + source: 'test source', + type: 'test type', + tags: [{ test_name: 'test value' }] } + end + + def exception_event + { exception: { message: @error.message, backtrace: @error.backtrace.join("\n\t") }, + klass: @error.class, + tags: [{ test_name: 'test value' }] } + end + end +end diff --git a/test/test_netuitive_logger.rb b/test/test_netuitive_logger.rb index 7f5b74c..44c1c95 100644 --- a/test/test_netuitive_logger.rb +++ b/test/test_netuitive_logger.rb @@ -1,9 +1,3 @@ -require 'test/unit' -require 'mocha/test_unit' -require 'netuitive_ruby_api' -require 'netuitive_ruby_api/config_manager' -require 'netuitive_ruby_api/netuitive_logger' - module NetuitiveRubyApi class NetuitiveLoggerTest def test_format_age diff --git a/test/test_netuitive_ruby_api.rb b/test/test_netuitive_ruby_api.rb index 75df5b0..9bfc0c5 100644 --- a/test/test_netuitive_ruby_api.rb +++ b/test/test_netuitive_ruby_api.rb @@ -2,32 +2,30 @@ module NetuitiveRubyApi class NetuitiveRubyAPITest < Test::Unit::TestCase def setup @netuitived_server = mock - NetuitiveRubyAPI.setup(@netuitived_server) - NetuitiveRubyApi::NetuitiveLogger.setup + @data_manager = mock + NetuitiveRubyAPI.setup + NetuitiveRubyAPI.data_manager = @data_manager + NetuitiveRubyAPI.netuitivedServer = @netuitived_server end def test_stop_server @netuitived_server.expects(:stopServer).once - thread = NetuitiveRubyAPI.stop_server - thread.join + NetuitiveRubyAPI.stop_server end def test_send_metrics @netuitived_server.expects(:sendMetrics).once - thread = NetuitiveRubyAPI.send_metrics - thread.join + NetuitiveRubyAPI.send_metrics end def test_clear_metrics @netuitived_server.expects(:clearMetrics).once - thread = NetuitiveRubyAPI.clear_metrics - thread.join + NetuitiveRubyAPI.clear_metrics end def test_add_sample - @netuitived_server.expects(:addSample).once.with('test.id', 5) - thread = NetuitiveRubyAPI.add_sample('test.id', 5) - thread.join + @data_manager.expects(:add_sample).once.with('test.id', 5) + NetuitiveRubyAPI.add_sample('test.id', 5) end def test_interval @@ -37,41 +35,35 @@ def test_interval end def test_event - @netuitived_server.expects(:event).once.with('test message', Time.new(2000, 1, 1, 1, 1, 1), 'Ruby Event', 'Info', 'Ruby Agent', 'INFO', nil) - thread = NetuitiveRubyAPI.event('test message', Time.new(2000, 1, 1, 1, 1, 1)) - thread.join + @data_manager.expects(:event).once.with('test message', Time.new(2000, 1, 1, 1, 1, 1), 'Ruby Event', 'Info', 'Ruby Agent', 'INFO', nil) + NetuitiveRubyAPI.event('test message', Time.new(2000, 1, 1, 1, 1, 1)) end def test_exception_event error = RuntimeError.new - @netuitived_server.expects(:exceptionEvent).once.with({ message: error.message }, nil, nil) - thread = NetuitiveRubyAPI.exception_event(error) - thread.join + @data_manager.expects(:exception_event).once.with(error, nil, nil) + NetuitiveRubyAPI.exception_event(error) begin raise 'test exception' rescue => e - @netuitived_server.expects(:exceptionEvent).once.with({ message: e.message, backtrace: e.backtrace.join("\n\t") }, nil, nil) - thread = NetuitiveRubyAPI.exception_event(e) - thread.join + @data_manager.expects(:exception_event).once.with(e, nil, nil) + NetuitiveRubyAPI.exception_event(e) end end def test_add_counter_sample - @netuitived_server.expects(:addCounterSample).once.with('test.id', 5) - thread = NetuitiveRubyAPI.add_counter_sample('test.id', 5) - thread.join + @data_manager.expects(:add_counter_sample).once.with('test.id', 5) + NetuitiveRubyAPI.add_counter_sample('test.id', 5) end def test_aggregate_metric - @netuitived_server.expects(:aggregateMetric).once.with('test.id', 5) - thread = NetuitiveRubyAPI.aggregate_metric('test.id', 5) - thread.join + @data_manager.expects(:aggregate_metric).once.with('test.id', 5) + NetuitiveRubyAPI.aggregate_metric('test.id', 5) end - def aggregate_counter_metric - @netuitived_server.expects(:aggregateCounterMetric).once.with('test.id', 5) - thread = NetuitiveRubyAPI.aggregate_counter_metric('test.id', 5) - thread.join + def test_aggregate_counter_metric + @data_manager.expects(:aggregate_counter_metric).once.with('test.id', 5) + NetuitiveRubyAPI.aggregate_counter_metric('test.id', 5) end end end