msm: ipa3: Changes to enhance find free pages from list
Adding changes to enhance find free pages from list,
1. When free page not available instead of keep trying
using temporary pages.
2. In tasklet search for free pages adding to head.
Change-Id: I991336167519322081faf8f82feec4e3f0d7b161
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c
index a4b154f..d5faa9d 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c
@@ -8669,6 +8669,11 @@
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
memset(ipa3_ctx->stats.page_recycle_cnt, 0,
sizeof(ipa3_ctx->stats.page_recycle_cnt));
+ ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0;
+ ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0;
+ ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0;
+ ipa3_ctx->stats.num_of_times_wq_reschd = 0;
+ ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0;
ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
ipa3_ctx->ee = resource_p->ee;
@@ -8881,6 +8886,10 @@
/* Initialize Page poll threshold. */
ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
+ /*Initialize number napi without prealloc buff*/
+ ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD;
+ ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME;
+
/* Use common page pool for Def/Coal pipe. */
if (ipa3_ctx->ipa_hw_type >= IPA_HW_v5_1)
ipa3_ctx->wan_common_page_pool = true;
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
index 07ad67e..e94c261 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
@@ -1611,15 +1611,23 @@
"COAL : Total number of packets replenished =%llu\n"
"COAL : Number of page recycled packets =%llu\n"
"COAL : Number of tmp alloc packets =%llu\n"
+ "COAL : Number of times tasklet scheduled =%llu\n"
"DEF : Total number of packets replenished =%llu\n"
"DEF : Number of page recycled packets =%llu\n"
- "DEF : Number of tmp alloc packets =%llu\n",
+ "DEF : Number of tmp alloc packets =%llu\n"
+ "DEF : Number of times tasklet scheduled =%llu\n"
+ "COMMON : Number of page recycled in tasklet =%llu\n"
+ "COMMON : Number of times free pages not found in tasklet =%llu\n",
ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
+ ipa3_ctx->stats.num_sort_tasklet_sched[0],
ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
- ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
+ ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc,
+ ipa3_ctx->stats.num_sort_tasklet_sched[1],
+ ipa3_ctx->stats.page_recycle_cnt_in_tasklet,
+ ipa3_ctx->stats.num_of_times_wq_reschd);
cnt += nbytes;
@@ -3011,6 +3019,68 @@
return count;
}
+static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file,
+ char __user *buf, size_t count, loff_t *ppos) {
+
+ int nbytes;
+ nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+ "page max napi without free page = %d\n",
+ ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+ return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+
+static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file,
+ const char __user *buf, size_t count, loff_t *ppos) {
+
+ int ret;
+ u8 ipa_max_napi_sort_page_thrshld = 0;
+
+ if (count >= sizeof(dbg_buff))
+ return -EFAULT;
+
+ ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld);
+ if(ret)
+ return ret;
+
+ ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld;
+
+ IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+
+ return count;
+}
+
+static ssize_t ipa3_read_page_wq_reschd_time(struct file *file,
+ char __user *buf, size_t count, loff_t *ppos) {
+
+ int nbytes;
+ nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+ "Page WQ reschduule time = %d\n",
+ ipa3_ctx->page_wq_reschd_time);
+ return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+
+static ssize_t ipa3_write_page_wq_reschd_time(struct file *file,
+ const char __user *buf, size_t count, loff_t *ppos) {
+
+ int ret;
+ u8 page_wq_reschd_time = 0;
+
+ if (count >= sizeof(dbg_buff))
+ return -EFAULT;
+
+ ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time);
+ if(ret)
+ return ret;
+
+ ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time;
+
+ IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time);
+
+ return count;
+}
+
static ssize_t ipa3_read_page_poll_threshold(struct file *file,
char __user *buf, size_t count, loff_t *ppos) {
@@ -3316,6 +3386,16 @@
"move_nat_table_to_ddr", IPA_WRITE_ONLY_MODE, NULL,{
.write = ipa3_write_nat_table_move,
}
+ }, {
+ "page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, {
+ .read = ipa3_read_page_wq_reschd_time,
+ .write = ipa3_write_page_wq_reschd_time,
+ }
+ }, {
+ "ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, {
+ .read = ipa3_read_ipa_max_napi_sort_page_thrshld,
+ .write = ipa3_write_ipa_max_napi_sort_page_thrshld,
+ }
},
};
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 5ae7ee8..c56ac54 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -143,6 +143,7 @@
static unsigned long tag_to_pointer_wa(uint64_t tag);
static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt);
static void ipa3_tasklet_rx_notify(unsigned long data);
+static void ipa3_tasklet_find_freepage(unsigned long data);
static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit);
static int ipa3_rmnet_ll_rx_poll(struct napi_struct *napi_rx, int budget);
@@ -1194,6 +1195,60 @@
return result;
}
+static void ipa3_schd_freepage_work(struct work_struct *work)
+{
+ struct delayed_work *dwork;
+ struct ipa3_sys_context *sys;
+
+ dwork = container_of(work, struct delayed_work, work);
+ sys = container_of(dwork, struct ipa3_sys_context, freepage_work);
+
+ IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n");
+
+ tasklet_schedule(&sys->tasklet_find_freepage);
+}
+
+static void ipa3_tasklet_find_freepage(unsigned long data)
+{
+ struct ipa3_sys_context *sys;
+ struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
+ struct ipa3_rx_pkt_wrapper *tmp = NULL;
+ struct page *cur_page;
+ int found_free_page = 0;
+ struct list_head temp_head;
+
+ sys = (struct ipa3_sys_context *)data;
+
+ INIT_LIST_HEAD(&temp_head);
+ spin_lock_bh(&sys->common_sys->spinlock);
+ list_for_each_entry_safe(rx_pkt, tmp,
+ &sys->page_recycle_repl->page_repl_head, link) {
+ cur_page = rx_pkt->page_data.page;
+ if (page_ref_count(cur_page) == 1) {
+ /* Found a free page. */
+ list_del_init(&rx_pkt->link);
+ list_add(&rx_pkt->link, &temp_head);
+ found_free_page++;
+ }
+ }
+ if (!found_free_page) {
+ /*Not found free page rescheduling tasklet after 2msec*/
+ IPADBG_LOW("Scheduling WQ not found free pages\n");
+ ++ipa3_ctx->stats.num_of_times_wq_reschd;
+ queue_delayed_work(sys->freepage_wq,
+ &sys->freepage_work,
+ msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time));
+ } else {
+ /*Allow to use pre-allocated buffers*/
+ list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head);
+ ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page;
+ IPADBG_LOW("found free pages count = %d\n", found_free_page);
+ atomic_set(&sys->common_sys->page_avilable, 1);
+ }
+ spin_unlock_bh(&sys->common_sys->spinlock);
+
+}
+
/**
* ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform
* IPA EP configuration
@@ -1286,6 +1341,9 @@
goto fail_wq2;
}
+ snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d",
+ sys_in->client);
+
INIT_LIST_HEAD(&ep->sys->head_desc_list);
INIT_LIST_HEAD(&ep->sys->rcycl_list);
INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list);
@@ -1302,6 +1360,16 @@
/* Use coalescing pipe PM handle for default pipe also*/
ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl;
} else if (IPA_CLIENT_IS_CONS(sys_in->client)) {
+ ep->sys->freepage_wq = alloc_workqueue(buff,
+ WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS |
+ WQ_HIGHPRI, 1);
+ if (!ep->sys->freepage_wq) {
+ IPAERR("failed to create freepage wq for client %d\n",
+ sys_in->client);
+ result = -EFAULT;
+ goto fail_wq3;
+ }
+
pm_reg.name = ipa_clients_strings[sys_in->client];
pm_reg.callback = ipa_pm_sys_pipe_cb;
pm_reg.user_data = ep->sys;
@@ -1430,6 +1498,7 @@
}
*clnt_hdl = ipa_ep_idx;
+ ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) {
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
@@ -1505,6 +1574,10 @@
atomic_set(&ep->sys->repl->head_idx, 0);
atomic_set(&ep->sys->repl->tail_idx, 0);
+ tasklet_init(&ep->sys->tasklet_find_freepage,
+ ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
+ INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
+ ep->sys->napi_sort_page_thrshld_cnt = 0;
ipa3_replenish_rx_page_cache(ep->sys);
ipa3_wq_page_repl(&ep->sys->repl_work);
} else {
@@ -1615,6 +1688,8 @@
fail_gen2:
ipa_pm_deregister(ep->sys->pm_hdl);
fail_pm:
+ destroy_workqueue(ep->sys->freepage_wq);
+fail_wq3:
destroy_workqueue(ep->sys->repl_wq);
fail_wq2:
destroy_workqueue(ep->sys->wq);
@@ -2377,6 +2452,7 @@
list_add_tail(&rx_pkt->link,
&sys->page_recycle_repl->page_repl_head);
}
+ atomic_set(&sys->common_sys->page_avilable, 1);
return;
@@ -2457,6 +2533,7 @@
int i = 0;
u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
+ spin_lock_bh(&sys->common_sys->spinlock);
list_for_each_entry_safe(rx_pkt, tmp,
&sys->page_recycle_repl->page_repl_head, link) {
if (i == LOOP_THRESHOLD)
@@ -2467,10 +2544,23 @@
page_ref_inc(cur_page);
list_del_init(&rx_pkt->link);
++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
+ sys->common_sys->napi_sort_page_thrshld_cnt = 0;
+ spin_unlock_bh(&sys->common_sys->spinlock);
return rx_pkt;
}
i++;
}
+ spin_unlock_bh(&sys->common_sys->spinlock);
+ IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n",
+ sys->common_sys->napi_sort_page_thrshld_cnt,
+ ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+ /*Scheduling tasklet to find the free page*/
+ if (sys->common_sys->napi_sort_page_thrshld_cnt >=
+ ipa3_ctx->ipa_max_napi_sort_page_thrshld) {
+ atomic_set(&sys->common_sys->page_avilable, 0);
+ tasklet_schedule(&sys->common_sys->tasklet_find_freepage);
+ ++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i];
+ }
return NULL;
}
@@ -2543,7 +2633,8 @@
while (rx_len_cached < sys->rx_pool_sz) {
/* check for an idle page that can be used */
- if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) {
+ if (atomic_read(&sys->common_sys->page_avilable) &&
+ ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) {
ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
} else {
@@ -4138,11 +4229,11 @@
IPAERR("notify->veid > GSI_VEID_MAX\n");
if (!rx_page.is_tmp_alloc) {
init_page_count(rx_page.page);
- spin_lock_bh(&rx_pkt->sys->spinlock);
+ spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element to head. */
list_add(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
- spin_unlock_bh(&rx_pkt->sys->spinlock);
+ spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
} else {
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
@@ -4170,11 +4261,11 @@
list_del_init(&rx_pkt->link);
if (!rx_page.is_tmp_alloc) {
init_page_count(rx_page.page);
- spin_lock_bh(&rx_pkt->sys->spinlock);
+ spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element to head. */
list_add(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
- spin_unlock_bh(&rx_pkt->sys->spinlock);
+ spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
} else {
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
@@ -4194,11 +4285,11 @@
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
} else {
- spin_lock_bh(&rx_pkt->sys->spinlock);
+ spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element back to tail. */
list_add_tail(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
- spin_unlock_bh(&rx_pkt->sys->spinlock);
+ spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
dma_sync_single_for_cpu(ipa3_ctx->pdev,
rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
@@ -5917,6 +6008,7 @@
return -EINVAL;
}
+ ep->sys->common_sys->napi_sort_page_thrshld_cnt++;
start_poll:
/*
* it is guaranteed we already have clock here.
@@ -6148,6 +6240,7 @@
return -EINVAL;
}
+ sys->napi_sort_page_thrshld_cnt++;
start_poll:
/*
* it is guaranteed we already have clock here.
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index 2654ccf..9bd1490 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -129,6 +129,8 @@
#define NTN3_CLIENTS_NUM 2
+#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3
+#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2
#define IPA_WDI2_OVER_GSI() (ipa3_ctx->ipa_wdi2_over_gsi \
&& (ipa_get_wdi_version() == IPA_WDI_2))
@@ -1197,6 +1199,10 @@
bool ext_ioctl_v2;
bool common_buff_pool;
struct ipa3_sys_context *common_sys;
+ struct tasklet_struct tasklet_find_freepage;
+ atomic_t page_avilable;
+ struct delayed_work freepage_work;
+ u32 napi_sort_page_thrshld_cnt;
/* ordering is important - mutable fields go above */
struct ipa3_ep_context *ep;
@@ -1210,6 +1216,7 @@
struct workqueue_struct *repl_wq;
struct ipa3_status_stats *status_stat;
u32 pm_hdl;
+ struct workqueue_struct *freepage_wq;
/* ordering is important - other immutable fields go below */
};
@@ -1560,6 +1567,9 @@
atomic_t num_buff_above_thresh_for_coal_pipe_notified;
atomic_t num_buff_below_thresh_for_def_pipe_notified;
atomic_t num_buff_below_thresh_for_coal_pipe_notified;
+ u64 num_sort_tasklet_sched[3];
+ u64 num_of_times_wq_reschd;
+ u64 page_recycle_cnt_in_tasklet;
};
/* offset for each stats */
@@ -2389,7 +2399,8 @@
int uc_act_tbl_total;
int uc_act_tbl_next_index;
int ipa_pil_load;
-
+ u32 ipa_max_napi_sort_page_thrshld;
+ u32 page_wq_reschd_time;
};
struct ipa3_plat_drv_res {