[MPlayer-dev-eng] [PATCH] Remove auto-inserted lavc if vd init fails

[Reimar Döffinger]

Hi,
On Sun, Feb 27, 2005 at 01:05:29AM -0600, Zoltan Hidvegi wrote:
> The attached patch removes the auto-inserted lavc if
> mpcodecs_config_vo fails.

IMHO the whole code needs to be reworked, e.g. it also select Y800
output format (grayscale) if that is the only YUV format supported in
hardware, and a lot more stupidities. Unfortunately this code is
duplicated somewhere (I think vf.c), so there are two places that are in
need of a fix :-(
I attached the patch I created back then, but it is no perfect and
doesn't fix your problem (yet).
If you have the time, please try to find a really good fix *g*

Greetings,
Reimar Döffinger
-------------- next part --------------
Index: libmpcodecs/vd.c
===================================================================
RCS file: /cvsroot/mplayer/main/libmpcodecs/vd.c,v
retrieving revision 1.79
diff -u -r1.79 vd.c
--- libmpcodecs/vd.c	1 Oct 2004 20:08:21 -0000	1.79
+++ libmpcodecs/vd.c	29 Oct 2004 11:41:08 -0000
@@ -140,14 +140,34 @@
 extern vd_functions_t* mpvdec; // FIXME!
 extern int divx_quality;
 
+static int is_fmt_better(unsigned old_fmt, int old_flags,
+                         unsigned fmt, int flags,
+                         unsigned preferred_fmt) {
+  if (!(flags & VFCAP_CSP_SUPPORTED))
+    return 0;
+  if (!(old_flags & VFCAP_CSP_SUPPORTED))
+    return 1;
+  if (fmt == IMGFMT_Y8 || fmt == IMGFMT_Y800)
+    return 0;
+  if (old_fmt == IMGFMT_Y8 || old_fmt == IMGFMT_Y800)
+    return 1;
+  if (!(flags & VFCAP_CSP_SUPPORTED_BY_HW) &&
+       (old_flags & VFCAP_CSP_SUPPORTED_BY_HW))
+    return 0;
+  if (!(old_flags & VFCAP_CSP_SUPPORTED_BY_HW) &&
+       (flags & VFCAP_CSP_SUPPORTED_BY_HW))
+    return 1;
+  return (fmt == preferred_fmt);
+}
+
 int mpcodecs_config_vo(sh_video_t *sh, int w, int h, unsigned int preferred_outfmt){
     int i,j;
     unsigned int out_fmt=0;
     int screen_size_x=0;//SCREEN_SIZE_X;
     int screen_size_y=0;//SCREEN_SIZE_Y;
 //    vo_functions_t* video_out=sh->video_out;
-    vf_instance_t* vf=sh->vfilter,*sc=NULL;
-    int palette=0;
+    vf_instance_t* vf=sh->vfilter,*sc=NULL, *pal = NULL;
+    int conv_try_nr = 0;
 
     if(!sh->disp_w || !sh->disp_h)
         mp_msg(MSGT_DECVIDEO,MSGL_WARN, MSGTR_CodecDidNotSet);
@@ -182,48 +202,64 @@
     }
 
     j=-1;
+    vo_flags = 0;
+    out_fmt = 0;
     for(i=0;i<CODECS_MAX_OUTFMT;i++){
 	int flags;
+	unsigned prev_fmt = out_fmt;
 	out_fmt=sh->codec->outfmt[i];
 	if(out_fmt==(unsigned int)0xFFFFFFFF) continue;
 	flags=vf->query_format(vf,out_fmt);
 	mp_msg(MSGT_CPLAYER,MSGL_DBG2,"vo_debug: query(%s) returned 0x%X (i=%d) \n",vo_format_name(out_fmt),flags,i);
-	if((flags&2) || (flags && j<0)){
+	if (is_fmt_better(prev_fmt, vo_flags, out_fmt, flags,
+	                   preferred_outfmt)) {
 	    // check (query) if codec really support this outfmt...
 	    sh->outfmtidx=j; // pass index to the control() function this way
 	    if(mpvdec->control(sh,VDCTRL_QUERY_FORMAT,&out_fmt)==CONTROL_FALSE){
 		mp_msg(MSGT_CPLAYER,MSGL_DBG2,"vo_debug: codec query_format(%s) returned FALSE\n",vo_format_name(out_fmt));
 		continue;
 	    }
-	    j=i; vo_flags=flags; if(flags&2) break;
-	} else
-	if(!palette && !(flags&3) && (out_fmt==IMGFMT_RGB8||out_fmt==IMGFMT_BGR8)){
-	    sh->outfmtidx=j; // pass index to the control() function this way
-	    if(mpvdec->control(sh,VDCTRL_QUERY_FORMAT,&out_fmt)!=CONTROL_FALSE)
-		palette=1;
+	    j=i; vo_flags=flags;
 	}
     }
     if(j<0){
-	// TODO: no match - we should use conversion...
-	if(strcmp(vf->info->name,"scale") && palette!=-1){
+	// no match or only Y8 csp matched and we did not yet try vf_scale
+	// Try palette, then scale, then scale + palette and MPEGPES last.
+	while (vf == sc || vf == pal) {
+	  // Remove any filters we added ourselves
+	  vf_instance_t *ve;
+	  ve = vf;
+	  vf = vf->next;
+	  vf_uninit_filter(ve);
+	}
+	pal = sc = NULL;
+	switch (conv_try_nr) {
+	 case 0:
+	  conv_try_nr++;
+	  if (strcmp(vf->info->name, "scale") != 0) {
 	    mp_msg(MSGT_DECVIDEO,MSGL_INFO,MSGTR_CouldNotFindColorspace);
 	    sc=vf=vf_open_filter(vf,"scale",NULL);
 	    goto csp_again;
-	} else
-	if(palette==1){
+	  }
+	   // scale is already there
+	 case 1:
+	  conv_try_nr++;
+	  if (strcmp(vf->info->name, "palette") != 0) {
 	    mp_msg(MSGT_DECVIDEO,MSGL_V,"vd: Trying -vf palette...\n");
-	    palette=-1;
-	    vf=vf_open_filter(vf,"palette",NULL);
+	    pal=vf=vf_open_filter(vf,"palette",NULL);
+	    goto csp_again;
+	  }
+	 case 2:
+	  conv_try_nr++;
+	  if (strcmp(vf->info->name, "scale") != 0) {
+	    mp_msg(MSGT_DECVIDEO,MSGL_V,"vd: Trying -vf palette, scale...\n");
+	    sc=vf=vf_open_filter(vf,"scale",NULL);
+	    pal=vf=vf_open_filter(vf,"palette",NULL);
 	    goto csp_again;
-	} else 
+	  }
+	 case 3:
 	{ // sws failed, if the last filter (vf_vo) support MPEGPES try to append vf_lavc
 	     vf_instance_t* vo, *vp = NULL, *ve;
-	     // Remove the scale filter if we added it ourself
-	     if(vf == sc) {
-	       ve = vf;
-	       vf = vf->next;
-	       vf_uninit_filter(ve);
-	     }
 	     // Find the last filter (vf_vo)
 	     for(vo = vf ; vo->next ; vo = vo->next)
 	       vp = vo;
@@ -234,6 +270,7 @@
 	       goto csp_again;
 	     }
 	}
+	}
 	mp_msg(MSGT_CPLAYER,MSGL_WARN,MSGTR_VOincompCodec);
 	sh->vf_inited=-1;
 	return 0;	// failed