fix(types): fix instance type when props type is incompatible with setup returned type (#7338)

close #5885

packages/dts-test/defineComponent.test-d.tsx

@@ -1472,6 +1472,31 @@ describe('slots', () => {
   expectType<Slots | undefined>(new comp2().$slots)
 })
 
+// #5885
+describe('should work when props type is incompatible with setup returned type ', () => {
+  type SizeType = 'small' | 'big'
+  const Comp = defineComponent({
+    props: {
+      size: {
+        type: String as PropType<SizeType>,
+        required: true
+      }
+    },
+    setup(props) {
+      expectType<SizeType>(props.size)
+      return {
+        size: 1
+      }
+    }
+  })
+  type CompInstance = InstanceType<typeof Comp>
+
+  const CompA = {} as CompInstance
+  expectType<ComponentPublicInstance>(CompA)
+  expectType<number>(CompA.size)
+  expectType<SizeType>(CompA.$props.size)
+})
+
 import {
   DefineComponent,
   ComponentOptionsMixin,

packages/runtime-core/src/apiDefineComponent.ts

@@ -70,8 +70,7 @@ export type DefineComponent<
     true,
     {},
     S
-  > &
-    Props
+  >
 > &
   ComponentOptionsBase<
     Props,

packages/runtime-core/src/componentPublicInstance.ts

@@ -15,7 +15,8 @@ import {
   isString,
   isFunction,
   UnionToIntersection,
-  Prettify
+  Prettify,
+  IfAny
 } from '@vue/shared'
 import {
   toRaw,
@@ -187,7 +188,6 @@ export type CreateComponentPublicInstance<
   I,
   S
 >
-
 // public properties exposed on the proxy, which is used as the render context
 // in templates (as `this` in the render option)
 export type ComponentPublicInstance<
@@ -226,7 +226,7 @@ export type ComponentPublicInstance<
       : (...args: any) => any,
     options?: WatchOptions
   ): WatchStopHandle
-} & P &
+} & IfAny<P, P, Omit<P, keyof ShallowUnwrapRef<B>>> &
   ShallowUnwrapRef<B> &
   UnwrapNestedRefs<D> &
   ExtractComputedReturns<C> &