zebra: use async dplane route updates

Enqueue updates to the dplane system; add a couple of stats.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

zebra/zebra_rib.c

@@ -1169,8 +1169,11 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
 {
 	struct nexthop *nexthop;
 	rib_table_info_t *info = srcdest_rnode_table_info(rn);
-	const struct prefix *p, *src_p;
 	struct zebra_vrf *zvrf = vrf_info_lookup(re->vrf_id);
+	const struct prefix *p, *src_p;
+	enum zebra_dplane_result ret;
+
+	rib_dest_t *dest = rib_dest_from_rnode(rn);
 
 	srcdest_rnode_prefixes(rn, &p, &src_p);
 
@@ -1202,23 +1205,39 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
 	if (old && (old != re) && (old->type != re->type))
 		zsend_route_notify_owner(old, p, ZAPI_ROUTE_BETTER_ADMIN_WON);
 
+	/* Update fib selection */
+	dest->selected_fib = re;
+
 	/*
 	 * Make sure we update the FPM any time we send new information to
 	 * the kernel.
 	 */
 	hook_call(rib_update, rn, "installing in kernel");
-	switch (kernel_route_rib(rn, p, src_p, old, re)) {
+
+	/* Send add or update */
+	if (old && (old != re)) {
+		ret = dplane_route_update(rn, re, old);
+	} else {
+		ret = dplane_route_add(rn, re);
+	}
+
+	switch (ret) {
 	case ZEBRA_DPLANE_REQUEST_QUEUED:
-		flog_err(
+		if (zvrf)
-			EC_ZEBRA_DP_INVALID_RC,
+			zvrf->installs_queued++;
-			"No current known DataPlane interfaces can return this, please fix");
 		break;
 	case ZEBRA_DPLANE_REQUEST_FAILURE:
-		flog_err(
+	{
-			EC_ZEBRA_DP_INSTALL_FAIL,
+		char str[SRCDEST2STR_BUFFER];
-			"No current known Rib Install Failure cases, please fix");
+
+		srcdest_rnode2str(rn, str, sizeof(str));
+		flog_err(EC_ZEBRA_DP_INSTALL_FAIL,
+			 "%u:%s: Failed to enqueue dataplane install",
+			 re->vrf_id, str);
 		break;
+	}
 	case ZEBRA_DPLANE_REQUEST_SUCCESS:
+		if (zvrf)
 			zvrf->installs++;
 		break;
 	}
@@ -1231,11 +1250,8 @@ void rib_uninstall_kernel(struct route_node *rn, struct route_entry *re)
 {
 	struct nexthop *nexthop;
 	rib_table_info_t *info = srcdest_rnode_table_info(rn);
-	const struct prefix *p, *src_p;
 	struct zebra_vrf *zvrf = vrf_info_lookup(re->vrf_id);
 
-	srcdest_rnode_prefixes(rn, &p, &src_p);
-
 	if (info->safi != SAFI_UNICAST) {
 		for (ALL_NEXTHOPS(re->ng, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
@@ -1244,20 +1260,25 @@ void rib_uninstall_kernel(struct route_node *rn, struct route_entry *re)
 
 	/*
 	 * Make sure we update the FPM any time we send new information to
-	 * the kernel.
+	 * the dataplane.
 	 */
 	hook_call(rib_update, rn, "uninstalling from kernel");
-	switch (kernel_route_rib(rn, p, src_p, re, NULL)) {
+
+	switch (dplane_route_delete(rn, re)) {
 	case ZEBRA_DPLANE_REQUEST_QUEUED:
-		flog_err(
+		if (zvrf)
-			EC_ZEBRA_DP_INVALID_RC,
+			zvrf->removals_queued++;
-			"No current known DataPlane interfaces can return this, please fix");
 		break;
 	case ZEBRA_DPLANE_REQUEST_FAILURE:
-		flog_err(
+	{
-			EC_ZEBRA_DP_INSTALL_FAIL,
+		char str[SRCDEST2STR_BUFFER];
-			"No current known RIB Install Failure cases, please fix");
+
+		srcdest_rnode2str(rn, str, sizeof(str));
+		flog_err(EC_ZEBRA_DP_INSTALL_FAIL,
+			 "%u:%s: Failed to enqueue dataplane uninstall",
+			 re->vrf_id, str);
 		break;
+	}
 	case ZEBRA_DPLANE_REQUEST_SUCCESS:
 		if (zvrf)
 			zvrf->removals++;
@@ -1272,6 +1293,7 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 {
 	rib_table_info_t *info = srcdest_rnode_table_info(rn);
 	rib_dest_t *dest = rib_dest_from_rnode(rn);
+	struct nexthop *nexthop;
 
 	if (dest && dest->selected_fib == re) {
 		if (info->safi == SAFI_UNICAST)
@@ -1283,6 +1305,11 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 
 		if (!RIB_SYSTEM_ROUTE(re))
 			rib_uninstall_kernel(rn, re);
+
+		dest->selected_fib = NULL;
+
+		for (ALL_NEXTHOPS(re->ng, nexthop))
+			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 	}
 
 	if (CHECK_FLAG(re->flags, ZEBRA_FLAG_SELECTED)) {
@@ -1926,16 +1953,18 @@ done:
 }
 
 /*
- * TODO - WIP
+ * TODO - WIP version of route-update processing after async dataplane
+ * update.
  */
 static void rib_process_after(dplane_ctx_h ctx)
 {
 	struct route_table *table = NULL;
+	struct zebra_vrf *zvrf = NULL;
 	struct route_node *rn = NULL;
 	struct route_entry *re = NULL, *old_re = NULL, *rib;
 	bool is_update = false;
 	struct nexthop *nexthop;
-	char dest_str[PREFIX_STRLEN];
+	char dest_str[PREFIX_STRLEN] = "";
 	dplane_op_e op;
 	enum zebra_dplane_result status;
 	const struct prefix *dest_pfx, *src_pfx;
@@ -1957,6 +1986,8 @@ static void rib_process_after(dplane_ctx_h ctx)
 		goto done;
 	}
 
+	zvrf = vrf_info_lookup(dplane_ctx_get_vrf(ctx));
+
 	dest_pfx = dplane_ctx_get_dest(ctx);
 
 	/* Note well: only capturing the prefix string if debug is enabled here;
@@ -1997,6 +2028,10 @@ static void rib_process_after(dplane_ctx_h ctx)
 		 */
 		if (status == ZEBRA_DPLANE_REQUEST_SUCCESS) {
 			zsend_route_notify_owner_ctx(ctx, ZAPI_ROUTE_REMOVED);
+
+			if (zvrf) {
+				zvrf->removals++;
+			}
 		} else {
 			zsend_route_notify_owner_ctx(ctx,
 						     ZAPI_ROUTE_FAIL_INSTALL);
@@ -2085,6 +2120,10 @@ static void rib_process_after(dplane_ctx_h ctx)
 				UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		}
 
+		if (zvrf) {
+			zvrf->installs++;
+		}
+
 		/* Redistribute */
 		/* TODO -- still calling the redist api using the route_entries,
 		 * and there's a corner-case here: if there's no client

zebra/zebra_vrf.h

@@ -133,6 +133,8 @@ struct zebra_vrf {
 	/* Route Installs */
 	uint64_t installs;
 	uint64_t removals;
+	uint64_t installs_queued;
+	uint64_t removals_queued;
 	uint64_t neigh_updates;
 	uint64_t lsp_installs;
 	uint64_t lsp_removals;