From 0f490fdab17ab1aadea9f73958fa1698326cc092 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Tue, 15 Sep 2015 14:59:05 -0500 Subject: [PATCH] FS-8130 add support for timestamp based counting for jitter buffer in audio mode --- src/include/switch_jitterbuffer.h | 3 +- src/mod/codecs/mod_opus/mod_opus.c | 2 +- src/mod/codecs/mod_silk/mod_silk.c | 2 +- src/switch_jitterbuffer.c | 122 +++++++++++++++++++++++++---- src/switch_rtp.c | 2 + 5 files changed, 114 insertions(+), 17 deletions(-) diff --git a/src/include/switch_jitterbuffer.h b/src/include/switch_jitterbuffer.h index 7ae0d04ece..08a71fae0a 100644 --- a/src/include/switch_jitterbuffer.h +++ b/src/include/switch_jitterbuffer.h @@ -47,7 +47,7 @@ SWITCH_BEGIN_EXTERN_C SWITCH_DECLARE(switch_status_t) switch_jb_create(switch_jb_t **jbp, switch_jb_type_t type, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool); SWITCH_DECLARE(switch_status_t) switch_jb_set_frames(switch_jb_t *jb, uint32_t min_frame_len, uint32_t max_frame_len); -SWITCH_DECLARE(switch_status_t) switch_jb_peek_frame(switch_jb_t *jb, uint16_t seq, int peek, switch_frame_t *frame); +SWITCH_DECLARE(switch_status_t) switch_jb_peek_frame(switch_jb_t *jb, uint32_t ts, uint16_t seq, int peek, switch_frame_t *frame); SWITCH_DECLARE(switch_status_t) switch_jb_get_frames(switch_jb_t *jb, uint32_t *min_frame_len, uint32_t *max_frame_len, uint32_t *cur_frame_len, uint32_t *highest_frame_len); SWITCH_DECLARE(switch_status_t) switch_jb_destroy(switch_jb_t **jbp); SWITCH_DECLARE(void) switch_jb_reset(switch_jb_t *jb); @@ -61,6 +61,7 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp SWITCH_DECLARE(uint32_t) switch_jb_pop_nack(switch_jb_t *jb); SWITCH_DECLARE(switch_status_t) switch_jb_get_packet_by_seq(switch_jb_t *jb, uint16_t seq, switch_rtp_packet_t *packet, switch_size_t *len); SWITCH_DECLARE(void) switch_jb_set_session(switch_jb_t *jb, switch_core_session_t *session); +SWITCH_DECLARE(void) switch_jb_ts_mode(switch_jb_t *jb, uint32_t samples_per_frame, uint32_t samples_per_second); SWITCH_DECLARE(void) switch_jb_set_flag(switch_jb_t *jb, switch_jb_flag_t flag); SWITCH_DECLARE(void) switch_jb_clear_flag(switch_jb_t *jb, switch_jb_flag_t flag); diff --git a/src/mod/codecs/mod_opus/mod_opus.c b/src/mod/codecs/mod_opus/mod_opus.c index fed3395e87..c50418aa87 100644 --- a/src/mod/codecs/mod_opus/mod_opus.c +++ b/src/mod/codecs/mod_opus/mod_opus.c @@ -597,7 +597,7 @@ static switch_status_t switch_opus_decode(switch_codec_t *codec, frame.data = buf; frame.buflen = sizeof(buf); - if (switch_jb_peek_frame(jb, codec->cur_frame->seq, 1, &frame)) { + if (switch_jb_peek_frame(jb, codec->cur_frame->timestamp, 0, 1, &frame)) { got_frame = 1; fec = 1; encoded_data = frame.data; diff --git a/src/mod/codecs/mod_silk/mod_silk.c b/src/mod/codecs/mod_silk/mod_silk.c index 040742da23..8da1c541c7 100644 --- a/src/mod/codecs/mod_silk/mod_silk.c +++ b/src/mod/codecs/mod_silk/mod_silk.c @@ -347,7 +347,7 @@ static switch_status_t switch_silk_decode(switch_codec_t *codec, frame.buflen = sizeof(buf); for (i = 1; i <= MAX_LBRR_DELAY; i++) { - if (switch_jb_peek_frame(jb, codec->cur_frame->seq, (uint16_t)i, &frame)) { + if (switch_jb_peek_frame(jb, codec->cur_frame->timestamp, 0, (uint16_t)i, &frame)) { SKP_Silk_SDK_search_for_LBRR(frame.data, (const int)frame.datalen, i, (SKP_uint8*) &context->recbuff, &context->reclen); if (context->reclen) { diff --git a/src/switch_jitterbuffer.c b/src/switch_jitterbuffer.c index 7041738812..489b568ccc 100644 --- a/src/switch_jitterbuffer.c +++ b/src/switch_jitterbuffer.c @@ -60,6 +60,9 @@ struct switch_jb_s { uint32_t highest_wrote_ts; uint32_t highest_wrote_seq; uint16_t target_seq; + uint32_t target_ts; + uint32_t last_target_ts; + uint16_t psuedo_seq; uint32_t visible_nodes; uint32_t complete_frames; uint32_t frame_len; @@ -73,6 +76,8 @@ struct switch_jb_s { uint32_t consec_good_count; uint32_t period_count; uint32_t dropped; + uint32_t samples_per_frame; + uint32_t samples_per_second; uint8_t write_init; uint8_t read_init; uint8_t debug_level; @@ -80,6 +85,7 @@ struct switch_jb_s { switch_size_t last_len; switch_inthash_t *missing_seq_hash; switch_inthash_t *node_hash; + switch_inthash_t *node_hash_ts; switch_mutex_t *mutex; switch_mutex_t *list_mutex; switch_memory_pool_t *pool; @@ -274,6 +280,10 @@ static inline void hide_node(switch_jb_node_t *node, switch_bool_t pop) } } + if (jb->node_hash_ts) { + switch_core_inthash_delete(jb->node_hash_ts, node->packet.header.ts); + } + switch_core_inthash_delete(jb->node_hash, node->packet.header.seq); switch_mutex_unlock(jb->list_mutex); @@ -388,13 +398,6 @@ static inline void jb_hit(switch_jb_t *jb) jb->consec_miss_count = 0; } -static inline void jb_miss(switch_jb_t *jb) -{ - jb->period_miss_count++; - jb->consec_miss_count++; - jb->consec_good_count = 0; -} - static void jb_frame_inc(switch_jb_t *jb, int i) { uint32_t old_frame_len = jb->frame_len; @@ -456,6 +459,13 @@ static void jb_frame_inc(switch_jb_t *jb, int i) } +static inline void jb_miss(switch_jb_t *jb) +{ + jb->period_miss_count++; + jb->consec_miss_count++; + jb->consec_good_count = 0; +} + static inline int verify_oldest_frame(switch_jb_t *jb) { switch_jb_node_t *lowest = jb_find_lowest_node(jb), *np = NULL; @@ -514,6 +524,10 @@ static inline void add_node(switch_jb_t *jb, switch_rtp_packet_t *packet, switch switch_core_inthash_insert(jb->node_hash, node->packet.header.seq, node); + if (jb->node_hash_ts) { + switch_core_inthash_insert(jb->node_hash_ts, node->packet.header.ts, node); + } + jb_debug(jb, (packet->header.m ? 1 : 2), "PUT packet last_ts:%u ts:%u seq:%u%s\n", ntohl(jb->highest_wrote_ts), ntohl(node->packet.header.ts), ntohs(node->packet.header.seq), packet->header.m ? " " : ""); @@ -521,7 +535,7 @@ static inline void add_node(switch_jb_t *jb, switch_rtp_packet_t *packet, switch - if (jb->write_init && ((abs(((int)htons(packet->header.seq) - htons(jb->highest_wrote_seq))) >= jb->max_frame_len) || + if (jb->write_init && ((abs(((int)ntohs(packet->header.seq) - ntohs(jb->highest_wrote_seq))) >= jb->max_frame_len) || (abs((int)((int64_t)ntohl(node->packet.header.ts) - (int64_t)ntohl(jb->highest_wrote_ts))) > (900000 * 5)))) { jb_debug(jb, 2, "%s", "CHANGE DETECTED, PUNT\n"); switch_jb_reset(jb); @@ -560,6 +574,24 @@ static inline void add_node(switch_jb_t *jb, switch_rtp_packet_t *packet, switch } } +static inline void increment_ts(switch_jb_t *jb) +{ + if (!jb->target_ts) return; + + jb->target_ts = htonl((ntohl(jb->target_ts) + jb->samples_per_frame)); + jb->psuedo_seq++; +} + +static inline void set_read_ts(switch_jb_t *jb, uint32_t ts) +{ + if (!ts) return; + + jb->last_target_ts = ts; + jb->target_ts = htonl((ntohl(jb->last_target_ts) + jb->samples_per_frame)); + jb->psuedo_seq++; +} + + static inline void increment_seq(switch_jb_t *jb) { jb->target_seq = htons((ntohs(jb->target_seq) + 1)); @@ -571,7 +603,7 @@ static inline void set_read_seq(switch_jb_t *jb, uint16_t seq) jb->target_seq = htons((ntohs(jb->last_target_seq) + 1)); } -static inline switch_status_t jb_next_packet(switch_jb_t *jb, switch_jb_node_t **nodep) +static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_node_t **nodep) { switch_jb_node_t *node = NULL; @@ -644,6 +676,48 @@ static inline switch_status_t jb_next_packet(switch_jb_t *jb, switch_jb_node_t * } + +static inline switch_status_t jb_next_packet_by_ts(switch_jb_t *jb, switch_jb_node_t **nodep) +{ + switch_jb_node_t *node = NULL; + + if (!jb->target_ts) { + if ((node = jb_find_lowest_node(jb))) { + jb_debug(jb, 2, "No target ts using ts: %u as a starting point\n", ntohl(node->packet.header.ts)); + } else { + jb_debug(jb, 1, "%s", "No nodes available....\n"); + } + jb_hit(jb); + } else if ((node = switch_core_inthash_find(jb->node_hash_ts, jb->target_ts))) { + jb_debug(jb, 2, "FOUND desired ts: %u\n", ntohl(jb->target_ts)); + jb_hit(jb); + } else { + jb_debug(jb, 2, "MISSING desired ts: %u\n", ntohl(jb->target_ts)); + jb_miss(jb); + increment_ts(jb); + } + + *nodep = node; + + if (node) { + set_read_ts(jb, node->packet.header.ts); + node->packet.header.seq = htons(jb->psuedo_seq); + return SWITCH_STATUS_SUCCESS; + } + + return SWITCH_STATUS_NOTFOUND; + +} + +static inline switch_status_t jb_next_packet(switch_jb_t *jb, switch_jb_node_t **nodep) +{ + if (jb->samples_per_frame) { + return jb_next_packet_by_ts(jb, nodep); + } else { + return jb_next_packet_by_seq(jb, nodep); + } +} + static inline void free_nodes(switch_jb_t *jb) { switch_mutex_lock(jb->list_mutex); @@ -651,6 +725,13 @@ static inline void free_nodes(switch_jb_t *jb) switch_mutex_unlock(jb->list_mutex); } +SWITCH_DECLARE(void) switch_jb_ts_mode(switch_jb_t *jb, uint32_t samples_per_frame, uint32_t samples_per_second) +{ + jb->samples_per_frame = samples_per_frame; + jb->samples_per_second = samples_per_second; + switch_core_inthash_init(&jb->node_hash_ts); +} + SWITCH_DECLARE(void) switch_jb_set_session(switch_jb_t *jb, switch_core_session_t *session) { jb->session = session; @@ -713,18 +794,27 @@ SWITCH_DECLARE(void) switch_jb_reset(switch_jb_t *jb) jb->period_good_count = 0; jb->consec_good_count = 0; jb->period_count = 0; + jb->target_ts = 0; + jb->last_target_ts = 0; switch_mutex_lock(jb->mutex); hide_nodes(jb); switch_mutex_unlock(jb->mutex); } -SWITCH_DECLARE(switch_status_t) switch_jb_peek_frame(switch_jb_t *jb, uint16_t seq, int peek, switch_frame_t *frame) +SWITCH_DECLARE(switch_status_t) switch_jb_peek_frame(switch_jb_t *jb, uint32_t ts, uint16_t seq, int peek, switch_frame_t *frame) { - uint16_t want_seq = seq + peek; - switch_jb_node_t *node; + switch_jb_node_t *node = NULL; - if ((node = switch_core_inthash_find(jb->node_hash, want_seq))) { + if (seq) { + uint16_t want_seq = seq + peek; + node = switch_core_inthash_find(jb->node_hash, want_seq); + } else if (ts && jb->samples_per_frame) { + uint32_t want_ts = ts + (peek * jb->samples_per_frame); + node = switch_core_inthash_find(jb->node_hash_ts, want_ts); + } + + if (node) { frame->seq = ntohs(node->packet.header.seq); frame->timestamp = ntohl(node->packet.header.ts); frame->m = node->packet.header.m; @@ -824,6 +914,10 @@ SWITCH_DECLARE(switch_status_t) switch_jb_destroy(switch_jb_t **jbp) } switch_core_inthash_destroy(&jb->node_hash); + if (jb->node_hash_ts) { + switch_core_inthash_destroy(&jb->node_hash_ts); + } + free_nodes(jb); if (jb->free_pool) { @@ -1039,8 +1133,8 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp default: if (jb->consec_miss_count > jb->frame_len) { switch_jb_reset(jb); - jb_debug(jb, 2, "%s", "Too many frames not found, RESIZE\n"); jb_frame_inc(jb, 1); + jb_debug(jb, 2, "%s", "Too many frames not found, RESIZE\n"); switch_goto_status(SWITCH_STATUS_RESTART, end); } else { jb_debug(jb, 2, "%s", "Frame not found suggest PLC\n"); diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 4ced6cbadb..c29e68e685 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -4060,6 +4060,8 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_jitter_buffer(switch_rtp_t * } else { status = switch_jb_create(&rtp_session->jb, SJB_AUDIO, queue_frames, max_queue_frames, rtp_session->pool); switch_jb_set_session(rtp_session->jb, rtp_session->session); + switch_jb_ts_mode(rtp_session->jb, samples_per_packet, samples_per_second); + //switch_jb_debug_level(rtp_session->jb, 10); } READ_DEC(rtp_session);